110 lines
5.7 KiB
Python
110 lines
5.7 KiB
Python
"""Run: python -m unittest discover -s tests -p demography_workbook_test.py"""
|
||
from decimal import Decimal
|
||
from pathlib import Path
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from zipfile import ZipFile
|
||
from xml.sax.saxutils import escape
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tools'))
|
||
from demography_workbook import age_group, parse
|
||
|
||
|
||
def workbook(path, rows):
|
||
cells = []
|
||
for number, row in enumerate(rows, 1):
|
||
content = ''
|
||
for column, value in row.items():
|
||
reference = f'{column}{number}'
|
||
content += f'<c r="{reference}" t="inlineStr"><is><t>{escape(value)}</t></is></c>'
|
||
cells.append(f'<row r="{number}">{content}</row>')
|
||
with ZipFile(path, 'w') as archive:
|
||
archive.writestr('xl/workbook.xml', '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="2017" sheetId="1" r:id="r1"/></sheets></workbook>')
|
||
archive.writestr('xl/_rels/workbook.xml.rels', '<Relationships><Relationship Id="r1" Target="worksheets/custom.xml"/></Relationships>')
|
||
archive.writestr('xl/worksheets/custom.xml', '<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>' + ''.join(cells) + '</sheetData></worksheet>')
|
||
|
||
|
||
class WorkbookTest(unittest.TestCase):
|
||
def fixture(self, rows):
|
||
with tempfile.TemporaryDirectory() as directory:
|
||
path = Path(directory) / 'synthetic.xlsx'
|
||
workbook(path, rows)
|
||
return parse(path)
|
||
|
||
def rows(self):
|
||
return [dict(A='QUADRO 1. População projectada por área de residência e sexo segundo idade. Manica, 2017'),
|
||
dict(A='Idade', C='TOTAL', F='URBANA', I='RURAL'),
|
||
dict(C='Total', D='Homens', E='Mulheres', F='Total', G='Homens', H='Mulheres', I='Total', J='Homens', K='Mulheres'),
|
||
dict(A='Total', C='100.125', D='40.025', E='60.100', F='40.025', G='20', H='20.025', I='60.100', J='20.025', K='40.075')]
|
||
|
||
def test_shifted_headers_exact_decimals_and_no_header_as_age_row(self):
|
||
report = self.fixture(self.rows())
|
||
self.assertEqual(9, report['record_count'])
|
||
self.assertEqual({}, report['issue_counts'])
|
||
self.assertEqual('100.125', report['records'][0]['original_value'])
|
||
self.assertEqual(Decimal('100.125'), Decimal(report['records'][0]['numeric_value']))
|
||
|
||
def test_conflicting_duplicates_are_reported(self):
|
||
rows = self.rows()
|
||
rows.append({**rows[-1], 'C': '200'})
|
||
self.assertEqual(1, self.fixture(rows)['issue_counts']['conflicting_value'])
|
||
|
||
def test_incomplete_cells_never_become_zero(self):
|
||
rows = self.rows()
|
||
del rows[-1]['K']
|
||
report = self.fixture(rows)
|
||
self.assertEqual(1, report['issue_counts']['missing_value'])
|
||
self.assertEqual(8, report['record_count'])
|
||
|
||
def test_sex_inconsistency_detected(self):
|
||
rows = self.rows()
|
||
rows[-1]['C'] = '999'
|
||
self.assertEqual(1, self.fixture(rows)['issue_counts']['sex_total_mismatch'])
|
||
|
||
def test_unknown_workbook_rejected(self):
|
||
with self.assertRaises(ValueError):
|
||
self.fixture([dict(A='unrecognised data')])
|
||
|
||
def test_city_da_and_municipal_headers_do_not_become_second_provinces(self):
|
||
for title, expected in [('Cidade da Matola', 'Matola'), ('Distrito Municipal KaMphumu', 'KaMphumu')]:
|
||
with self.subTest(title=title):
|
||
rows = self.rows()
|
||
rows += [dict(A='QUADRO 2. População projectada por área de residência e sexo segundo idade. '+title+', 2017'), rows[-1].copy()]
|
||
report = self.fixture(rows)
|
||
district = [r for r in report['records'] if r['level'] == 'district']
|
||
self.assertEqual(9, len(district))
|
||
self.assertEqual(expected, district[0]['territory'])
|
||
self.assertEqual('Manica', district[0]['province'])
|
||
self.assertEqual([title], report['territory_headers']['Manica|'+expected+'|district'])
|
||
|
||
def test_formula_uses_only_the_original_numeric_cache_and_reports_provenance(self):
|
||
for cache in ['100.125', '']:
|
||
with self.subTest(cache=cache), tempfile.TemporaryDirectory() as directory:
|
||
path = Path(directory) / 'cached.xlsx'
|
||
workbook(path, self.rows())
|
||
with ZipFile(path) as source:
|
||
members = {n: source.read(n) for n in source.namelist()}
|
||
xml = members['xl/worksheets/custom.xml'].decode()
|
||
xml = xml.replace('<c r="C4" t="inlineStr"><is><t>100.125</t></is></c>', '<c r="C4"><f>SUM(D4:E4)</f><v>'+cache+'</v></c>')
|
||
members['xl/worksheets/custom.xml'] = xml.encode()
|
||
with ZipFile(path, 'w') as target:
|
||
for name, content in members.items(): target.writestr(name, content)
|
||
report = parse(path)
|
||
if cache:
|
||
self.assertEqual('100.125', report['records'][0]['original_value'])
|
||
self.assertEqual(1, report['issue_counts']['cached_formula_value'])
|
||
self.assertNotIn('invalid_numeric_or_formula', report['issue_counts'])
|
||
self.assertEqual('SUM(D4:E4)', report['issues'][0]['formula'])
|
||
else:
|
||
self.assertEqual(1, report['issue_counts']['invalid_numeric_or_formula'])
|
||
self.assertEqual(8, report['record_count'])
|
||
|
||
def test_age_labels(self):
|
||
self.assertEqual('15-64', age_group('População em idade activa (15–64 anos)'))
|
||
self.assertEqual('65+', age_group('População idosa (65 ou mais anos)'))
|
||
|
||
|
||
if __name__ == '__main__':
|
||
unittest.main()
|