80 lines
3.9 KiB
Python
80 lines
3.9 KiB
Python
"""Preserve an INE PDF and prepare a review package; never guess table joins.
|
|
|
|
The 2012 INE PDF contains independently wrapped columns. A line-based merge
|
|
would associate valid codes with the wrong district. This tool deliberately
|
|
exports the extracted lines for review instead of manufacturing such joins.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
|
|
|
|
def prepare(source, destination, executable, source_url, source_version):
|
|
source = Path(source).resolve(strict=True)
|
|
destination = Path(destination).resolve()
|
|
original = source.read_bytes()
|
|
if len(original) > 20 * 1024 * 1024 or not original.startswith(b"%PDF-"):
|
|
raise ValueError("Expected a PDF of at most 20 MiB")
|
|
# Never overwrite an earlier review or its preserved original.
|
|
destination.mkdir(parents=True, exist_ok=False)
|
|
(destination / "original.pdf").write_bytes(original)
|
|
version = subprocess.run([executable, "-v"], capture_output=True, timeout=10, check=False)
|
|
# Xpdf 4.00 exits 99 after printing its version; Poppler exits zero.
|
|
if version.returncode not in (0, 99) or b"pdftotext version" not in version.stdout + version.stderr:
|
|
raise RuntimeError("Could not identify the pdftotext executable")
|
|
subprocess.run(
|
|
[executable, "-enc", "UTF-8", "-layout", str(destination / "original.pdf"), str(destination / "layout.txt")],
|
|
capture_output=True, timeout=60, check=True,
|
|
)
|
|
extracted = (destination / "layout.txt").read_text(encoding="utf-8")
|
|
counts = {}
|
|
records = 0
|
|
with (destination / "review.csv").open("w", encoding="utf-8", newline="") as stream:
|
|
writer = csv.writer(stream, delimiter=";")
|
|
writer.writerow(["page", "line", "extracted_text", "issue"])
|
|
for page, contents in enumerate(extracted.split("\f"), 1):
|
|
for line, text in enumerate(contents.splitlines(), 1):
|
|
if not text.strip():
|
|
continue
|
|
records += 1
|
|
# Preserve text exactly. A code occurrence is not a joined territory.
|
|
code = re.match(r"^\s*([0-9]{6})\s", text)
|
|
if code:
|
|
prefix = code.group(1)[:2]
|
|
counts[prefix] = counts.get(prefix, 0) + 1
|
|
writer.writerow([page, line, text, "manual_column_alignment_required"])
|
|
report = {
|
|
"schema_version": 1,
|
|
"source_url": source_url,
|
|
"source_version_declared": source_version,
|
|
"sha256": hashlib.sha256(original).hexdigest(),
|
|
"extraction_sha256": hashlib.sha256((destination / "layout.txt").read_bytes()).hexdigest(),
|
|
"extractor": (version.stdout + version.stderr).decode("utf-8", errors="replace").strip(),
|
|
"status": "requires_steward_review",
|
|
"license_status": "unverified",
|
|
"nonempty_extracted_lines": records,
|
|
"lines_requiring_review": records,
|
|
"automatically_accepted_territories": 0,
|
|
"observed_admin_post_code_lines_by_prefix": dict(sorted(counts.items())),
|
|
"source_totals_comparison": "pending_independent_visual_review",
|
|
"notice": "Code occurrences are not validated totals. No administrative relation or locality was inferred."
|
|
}
|
|
(destination / "report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
return report
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("source")
|
|
parser.add_argument("destination")
|
|
parser.add_argument("--pdftotext", required=True)
|
|
parser.add_argument("--source-url", required=True)
|
|
parser.add_argument("--source-version", required=True)
|
|
args = parser.parse_args()
|
|
report = prepare(args.source, args.destination, args.pdftotext, args.source_url, args.source_version)
|
|
print(json.dumps({"status": report["status"], "sha256": report["sha256"], "review_lines": report["lines_requiring_review"]}))
|