varias alteracoes
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
"""Preserve an INE PDF and prepare a review package; never guess table joins.
|
||||
|
||||
The 2012 INE PDF contains independently wrapped columns. A line-based merge
|
||||
would associate valid codes with the wrong district. This tool deliberately
|
||||
exports the extracted lines for review instead of manufacturing such joins.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
|
||||
def prepare(source, destination, executable, source_url, source_version):
|
||||
source = Path(source).resolve(strict=True)
|
||||
destination = Path(destination).resolve()
|
||||
original = source.read_bytes()
|
||||
if len(original) > 20 * 1024 * 1024 or not original.startswith(b"%PDF-"):
|
||||
raise ValueError("Expected a PDF of at most 20 MiB")
|
||||
# Never overwrite an earlier review or its preserved original.
|
||||
destination.mkdir(parents=True, exist_ok=False)
|
||||
(destination / "original.pdf").write_bytes(original)
|
||||
version = subprocess.run([executable, "-v"], capture_output=True, timeout=10, check=False)
|
||||
# Xpdf 4.00 exits 99 after printing its version; Poppler exits zero.
|
||||
if version.returncode not in (0, 99) or b"pdftotext version" not in version.stdout + version.stderr:
|
||||
raise RuntimeError("Could not identify the pdftotext executable")
|
||||
subprocess.run(
|
||||
[executable, "-enc", "UTF-8", "-layout", str(destination / "original.pdf"), str(destination / "layout.txt")],
|
||||
capture_output=True, timeout=60, check=True,
|
||||
)
|
||||
extracted = (destination / "layout.txt").read_text(encoding="utf-8")
|
||||
counts = {}
|
||||
records = 0
|
||||
with (destination / "review.csv").open("w", encoding="utf-8", newline="") as stream:
|
||||
writer = csv.writer(stream, delimiter=";")
|
||||
writer.writerow(["page", "line", "extracted_text", "issue"])
|
||||
for page, contents in enumerate(extracted.split("\f"), 1):
|
||||
for line, text in enumerate(contents.splitlines(), 1):
|
||||
if not text.strip():
|
||||
continue
|
||||
records += 1
|
||||
# Preserve text exactly. A code occurrence is not a joined territory.
|
||||
code = re.match(r"^\s*([0-9]{6})\s", text)
|
||||
if code:
|
||||
prefix = code.group(1)[:2]
|
||||
counts[prefix] = counts.get(prefix, 0) + 1
|
||||
writer.writerow([page, line, text, "manual_column_alignment_required"])
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"source_url": source_url,
|
||||
"source_version_declared": source_version,
|
||||
"sha256": hashlib.sha256(original).hexdigest(),
|
||||
"extraction_sha256": hashlib.sha256((destination / "layout.txt").read_bytes()).hexdigest(),
|
||||
"extractor": (version.stdout + version.stderr).decode("utf-8", errors="replace").strip(),
|
||||
"status": "requires_steward_review",
|
||||
"license_status": "unverified",
|
||||
"nonempty_extracted_lines": records,
|
||||
"lines_requiring_review": records,
|
||||
"automatically_accepted_territories": 0,
|
||||
"observed_admin_post_code_lines_by_prefix": dict(sorted(counts.items())),
|
||||
"source_totals_comparison": "pending_independent_visual_review",
|
||||
"notice": "Code occurrences are not validated totals. No administrative relation or locality was inferred."
|
||||
}
|
||||
(destination / "report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("source")
|
||||
parser.add_argument("destination")
|
||||
parser.add_argument("--pdftotext", required=True)
|
||||
parser.add_argument("--source-url", required=True)
|
||||
parser.add_argument("--source-version", required=True)
|
||||
args = parser.parse_args()
|
||||
report = prepare(args.source, args.destination, args.pdftotext, args.source_url, args.source_version)
|
||||
print(json.dumps({"status": report["status"], "sha256": report["sha256"], "review_lines": report["lines_requiring_review"]}))
|
||||
Reference in New Issue
Block a user