293 lines
18 KiB
PHP
293 lines
18 KiB
PHP
<?php
|
|
|
|
namespace App\Console\Commands;
|
|
|
|
use App\Jobs\ScanDocumentForMalware;
|
|
use App\Models\User;
|
|
use Illuminate\Console\Command;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Facades\Schema;
|
|
use Illuminate\Support\Facades\Storage;
|
|
use Illuminate\Support\Str;
|
|
use RuntimeException;
|
|
use Throwable;
|
|
|
|
class ImportDocumentBatch extends Command
|
|
{
|
|
protected $signature = 'documents:import-batch {manifest : JSON manifest inside storage/app/private/document-import-staging} {--apply : Register private drafts and queue malware scans} {--actor= : Existing user ID responsible for an applied import}';
|
|
|
|
protected $description = 'Preview or idempotently import a document batch into the existing private library';
|
|
|
|
private const MAX_BYTES = 10485760;
|
|
|
|
public function handle(): int
|
|
{
|
|
$staging = storage_path('app/private/document-import-staging');
|
|
$manifestPath = realpath($staging.DIRECTORY_SEPARATOR.$this->argument('manifest'));
|
|
$stagingReal = realpath($staging);
|
|
if (! $manifestPath || ! $stagingReal || ! str_starts_with(strtolower($manifestPath), strtolower($stagingReal.DIRECTORY_SEPARATOR)) || ! is_file($manifestPath)) {
|
|
$this->error('Manifest is missing or outside the private document-import staging directory.');
|
|
|
|
return self::FAILURE;
|
|
}
|
|
|
|
foreach ([
|
|
'documents' => ['id', 'source_system', 'external_id', 'version_no', 'data_classification', 'file_name', 'media_type', 'storage_key', 'sha256', 'status', 'size_bytes', 'issuer', 'issued_on', 'uploaded_by', 'rights_status', 'original_source', 'malware_scan_status'],
|
|
'portal_resources' => ['document_id', 'title', 'category', 'topic', 'published', 'source_reference', 'approved_by', 'description', 'document_type', 'document_number', 'document_date', 'source_url', 'language', 'version_label', 'source_confirmation', 'legal_status', 'rights_status', 'document_identity_key', 'metadata_version', 'created_by'],
|
|
] as $table => $columns) {
|
|
if (! Schema::hasColumns($table, $columns)) {
|
|
$this->error('Document-library schema is incomplete; run the documented schema compatibility check before importing.');
|
|
|
|
return self::FAILURE;
|
|
}
|
|
}
|
|
|
|
try {
|
|
$manifest = json_decode((string) file_get_contents($manifestPath), true, 512, JSON_THROW_ON_ERROR);
|
|
} catch (Throwable) {
|
|
$this->error('Manifest is not valid JSON.');
|
|
|
|
return self::FAILURE;
|
|
}
|
|
if (! is_array($manifest) || ! isset($manifest['documents']) || ! is_array($manifest['documents']) || count($manifest['documents']) > 100) {
|
|
$this->error('Manifest must contain at most 100 documents in a documents array.');
|
|
|
|
return self::FAILURE;
|
|
}
|
|
|
|
$apply = (bool) $this->option('apply');
|
|
$actorId = $this->option('actor');
|
|
if ($apply) {
|
|
// CLI attribution is not an authenticated active installation. Rehearsals stay isolated.
|
|
$target = DB::selectOne('SELECT current_database() AS database,inet_server_port() AS port');
|
|
if (! app()->environment('testing') || ! str_ends_with($target->database, '_test') || (int) $target->port !== 55441
|
|
|| config('database.connections.pgsql.host') !== '127.0.0.1' || config('database.connections.pgsql.url')) {
|
|
$this->error('CLI apply is restricted to isolated QA. Active imports require the authenticated activation mechanism; --actor is attribution only.');
|
|
|
|
return self::FAILURE;
|
|
}
|
|
}
|
|
if ($apply && (! is_numeric($actorId) || ! DB::table('users')->where('id', (int) $actorId)->exists())) {
|
|
$this->error('Apply requires --actor=<existing-user-id>; no author identity will be fabricated.');
|
|
|
|
return self::FAILURE;
|
|
}
|
|
if ($apply && ! User::findOrFail((int) $actorId)->can('manage-portal')) {
|
|
$this->error('QA import requires an authorized library manager; no permission is granted by naming an account.');
|
|
|
|
return self::FAILURE;
|
|
}
|
|
$disk = Storage::disk('local');
|
|
$seenHashes = [];
|
|
$seenExternalIds = [];
|
|
$dispatch = [];
|
|
$failures = 0;
|
|
|
|
foreach ($manifest['documents'] as $index => $record) {
|
|
$label = is_array($record) ? (string) ($record['external_id'] ?? '#'.($index + 1)) : '#'.($index + 1);
|
|
try {
|
|
$validated = $this->validateRecord($record, $stagingReal);
|
|
$hash = $validated['sha256'];
|
|
if (isset($seenHashes[$hash])) {
|
|
$this->line($label.' | duplicate bytes within batch; references item '.$seenHashes[$hash]);
|
|
|
|
continue;
|
|
}
|
|
$seenHashes[$hash] = $label;
|
|
if (isset($seenExternalIds[$validated['external_id']])) {
|
|
$this->warn($label.' | external_id already occurs in this batch with different bytes; identity review required');
|
|
|
|
continue;
|
|
}
|
|
$seenExternalIds[$validated['external_id']] = $hash;
|
|
|
|
$existing = DB::table('documents')->where('sha256', $hash)->whereNull('deleted_at')->first();
|
|
if ($existing) {
|
|
$this->line($label.' | already imported by SHA-256; document='.$existing->id.'; no duplicate');
|
|
|
|
continue;
|
|
}
|
|
$externalMatch = DB::table('documents')->where('source_system', 'document-batch-import')
|
|
->where('external_id', $validated['external_id'])->whereNull('deleted_at')->first();
|
|
if ($externalMatch) {
|
|
$this->warn($label.' | external_id already belongs to document '.$externalMatch->id.' with different bytes; version review required');
|
|
|
|
continue;
|
|
}
|
|
|
|
$identity = $this->identity($validated);
|
|
$identityMatch = DB::table('portal_resources')->where('document_identity_key', $identity)->first();
|
|
if ($identityMatch) {
|
|
$this->warn($label.' | identity matches resource '.$identityMatch->id.' but bytes differ; version review required');
|
|
|
|
continue;
|
|
}
|
|
|
|
$this->line($label.' | '.($apply ? 'will import as private draft' : 'preview: private draft')." | bytes={$validated['size_bytes']} | sha256={$hash}");
|
|
if (! $apply) {
|
|
continue;
|
|
}
|
|
|
|
$documentId = (string) Str::uuid();
|
|
$storageKey = 'portal-documents/'.$documentId;
|
|
$source = fopen($validated['source_path'], 'rb');
|
|
if (! is_resource($source)) {
|
|
throw new RuntimeException('Source could not be opened.');
|
|
}
|
|
try {
|
|
$stored = $disk->put($storageKey, $source);
|
|
} finally {
|
|
fclose($source);
|
|
}
|
|
$storedPath = $disk->path($storageKey);
|
|
$storedHash = $stored && is_file($storedPath) ? hash_file('sha256', $storedPath) : false;
|
|
if (! $stored || ! is_string($storedHash) || ! hash_equals($hash, $storedHash)) {
|
|
$disk->delete($storageKey);
|
|
throw new RuntimeException('Private copy failed the checksum verification.');
|
|
}
|
|
|
|
try {
|
|
$resourceId = DB::transaction(function () use ($validated, $documentId, $storageKey, $hash, $identity, $actorId): int {
|
|
DB::select('SELECT pg_advisory_xact_lock(hashtext(?))', ['document-batch:'.$hash]);
|
|
DB::select('SELECT pg_advisory_xact_lock(hashtext(?))', ['document-batch-identity:'.$identity]);
|
|
if (DB::table('documents')->where('sha256', $hash)->whereNull('deleted_at')->exists()) {
|
|
throw new RuntimeException('Matching bytes were registered concurrently; rerun preview.');
|
|
}
|
|
if (DB::table('portal_resources')->where('document_identity_key', $identity)->exists()) {
|
|
throw new RuntimeException('A matching document identity was registered concurrently; version review required.');
|
|
}
|
|
$now = now();
|
|
DB::table('documents')->insert([
|
|
'id' => $documentId, 'source_system' => 'document-batch-import', 'external_id' => $validated['external_id'],
|
|
'version_no' => 1, 'data_classification' => 'internal', 'file_name' => $validated['file_name'],
|
|
'media_type' => $validated['media_type'], 'storage_key' => $storageKey, 'sha256' => $hash,
|
|
'status' => 'quarantined', 'size_bytes' => $validated['size_bytes'], 'issuer' => $validated['issuer'],
|
|
'issued_on' => $validated['document_date'], 'uploaded_by' => (int) $actorId, 'rights_status' => 'unconfirmed',
|
|
'original_source' => $validated['source_url'], 'malware_scan_status' => 'pending',
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$resourceId = DB::table('portal_resources')->insertGetId([
|
|
'document_id' => $documentId, 'title' => $validated['title'], 'category' => $validated['category'],
|
|
'topic' => $validated['topic'], 'legislation' => $validated['legislation'], 'essential' => false,
|
|
'published' => false, 'source_reference' => $validated['source_reference'], 'approved_by' => null,
|
|
'description' => $validated['description'], 'document_type' => $validated['document_type'],
|
|
'document_number' => $validated['document_number'], 'document_date' => $validated['document_date'],
|
|
'source_url' => $validated['source_url'], 'language' => $validated['language'],
|
|
'version_label' => $validated['version_label'], 'territorial_scope' => 'unknown',
|
|
'source_confirmation' => 'unconfirmed', 'legal_status' => 'unknown', 'rights_status' => 'unconfirmed',
|
|
'document_identity_key' => $identity, 'metadata_version' => 1, 'created_by' => (int) $actorId,
|
|
'last_reviewed_by' => null, 'independent_review' => false, 'created_at' => $now, 'updated_at' => $now,
|
|
]);
|
|
$resource = DB::table('portal_resources')->where('id', $resourceId)->first();
|
|
DB::table('portal_resource_revisions')->insert([
|
|
'resource_id' => $resourceId, 'revision_no' => 1, 'change_type' => 'created',
|
|
'snapshot' => json_encode(['resource' => $resource, 'associations' => []], JSON_THROW_ON_ERROR),
|
|
'actor_user_id' => (int) $actorId, 'created_at' => $now,
|
|
]);
|
|
DB::table('audit_events')->insert([
|
|
'id' => (string) Str::uuid(), 'source_system' => 'document-batch-import',
|
|
'external_id' => $validated['external_id'], 'data_classification' => 'internal',
|
|
'actor_user_id' => (int) $actorId, 'entity_type' => 'document', 'entity_id' => $documentId,
|
|
'action' => 'private_document_batch_imported',
|
|
'after_data' => json_encode(['sha256' => $hash, 'resource_id' => $resourceId, 'published' => false], JSON_THROW_ON_ERROR),
|
|
'occurred_at' => $now, 'created_at' => $now,
|
|
]);
|
|
|
|
return $resourceId;
|
|
});
|
|
$dispatch[] = $documentId;
|
|
$this->info($label.' | private draft created | resource='.$resourceId.'; pending scan; not published');
|
|
} catch (Throwable $exception) {
|
|
$disk->delete($storageKey);
|
|
throw $exception;
|
|
}
|
|
} catch (Throwable $exception) {
|
|
$this->error($label.' | not imported: '.$exception->getMessage());
|
|
$failures++;
|
|
}
|
|
}
|
|
|
|
foreach ($dispatch as $documentId) {
|
|
ScanDocumentForMalware::dispatch($documentId)->afterCommit();
|
|
}
|
|
$this->line($apply ? 'New scans queued: '.count($dispatch).'.' : 'Preview only; database and storage unchanged.');
|
|
|
|
return $failures > 0 ? self::FAILURE : self::SUCCESS;
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
private function validateRecord(mixed $record, string $stagingRoot): array
|
|
{
|
|
if (! is_array($record)) {
|
|
throw new RuntimeException('Each documents entry must be an object.');
|
|
}
|
|
foreach (['external_id', 'relative_path', 'title', 'issuer'] as $required) {
|
|
if (! isset($record[$required]) || ! is_string($record[$required]) || trim($record[$required]) === '') {
|
|
throw new RuntimeException('Required manifest field is missing: '.$required.'.');
|
|
}
|
|
}
|
|
if (strlen($record['external_id']) > 120 || strlen($record['title']) > 250 || strlen($record['issuer']) > 180) {
|
|
throw new RuntimeException('A metadata value exceeds its permitted length.');
|
|
}
|
|
if (isset($record['file_name']) && (! is_string($record['file_name']) || strlen(basename($record['file_name'])) > 255)) {
|
|
throw new RuntimeException('File name exceeds its permitted length.');
|
|
}
|
|
if (isset($record['document_number']) && (! is_string($record['document_number']) || strlen($record['document_number']) > 120)) {
|
|
throw new RuntimeException('Document number exceeds its permitted length.');
|
|
}
|
|
if (isset($record['document_date'])) {
|
|
if (! is_string($record['document_date']) || ! preg_match('/^(\d{4})-(\d{2})-(\d{2})$/', $record['document_date'], $dateParts)
|
|
|| ! checkdate((int) $dateParts[2], (int) $dateParts[3], (int) $dateParts[1])) {
|
|
throw new RuntimeException('Document date must use a valid YYYY-MM-DD value.');
|
|
}
|
|
}
|
|
if (isset($record['document_type']) && ! in_array($record['document_type'], ['legislation', 'procedure', 'official_form', 'digital_adaptation', 'guidance_template', 'guide', 'study', 'reference_data', 'unknown'], true)) {
|
|
throw new RuntimeException('Document type is not supported by the catalogue.');
|
|
}
|
|
|
|
$path = realpath($stagingRoot.DIRECTORY_SEPARATOR.$record['relative_path']);
|
|
if (! $path || ! str_starts_with(strtolower($path), strtolower($stagingRoot.DIRECTORY_SEPARATOR)) || ! is_file($path)) {
|
|
throw new RuntimeException('Source file is missing or outside the private staging directory.');
|
|
}
|
|
$size = filesize($path);
|
|
if (! is_int($size) || $size < 1 || $size > self::MAX_BYTES) {
|
|
throw new RuntimeException('File size is outside the 1 byte to 10 MiB import limit.');
|
|
}
|
|
$mime = (new \finfo(FILEINFO_MIME_TYPE))->file($path);
|
|
$allowed = ['application/pdf', 'text/plain', 'text/csv', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'image/png', 'image/jpeg', 'image/webp'];
|
|
if (! is_string($mime) || ! in_array($mime, $allowed, true)) {
|
|
throw new RuntimeException('Detected media type is not supported by the document library.');
|
|
}
|
|
$sha = hash_file('sha256', $path);
|
|
if (! is_string($sha)) {
|
|
throw new RuntimeException('File hash could not be calculated.');
|
|
}
|
|
|
|
$sourceUrl = $record['source_url'] ?? null;
|
|
if ($sourceUrl !== null && (! is_string($sourceUrl) || strlen($sourceUrl) > 2000 || ! filter_var($sourceUrl, FILTER_VALIDATE_URL) || strtolower((string) parse_url($sourceUrl, PHP_URL_SCHEME)) !== 'https')) {
|
|
throw new RuntimeException('Source URL, when supplied, must be a valid HTTPS URL.');
|
|
}
|
|
|
|
return [
|
|
...$record, 'source_path' => $path, 'file_name' => basename((string) ($record['file_name'] ?? $path)),
|
|
'size_bytes' => $size, 'media_type' => $mime, 'sha256' => $sha,
|
|
'category' => (string) ($record['category'] ?? 'general'), 'topic' => (string) ($record['topic'] ?? 'unclassified'),
|
|
'legislation' => (bool) ($record['legislation'] ?? false), 'document_type' => (string) ($record['document_type'] ?? 'unknown'),
|
|
'document_number' => $record['document_number'] ?? null, 'document_date' => $record['document_date'] ?? null,
|
|
'language' => (string) ($record['language'] ?? 'unknown'), 'version_label' => $record['version_label'] ?? null,
|
|
'source_url' => $record['source_url'] ?? null, 'source_reference' => $record['source_reference'] ?? 'Origem por completar.',
|
|
'description' => $record['description'] ?? 'Documento importado para revisão documental.',
|
|
];
|
|
}
|
|
|
|
/** @param array<string, mixed> $record */
|
|
private function identity(array $record): string
|
|
{
|
|
return hash('sha256', mb_strtolower(implode('|', array_map('trim', [
|
|
(string) $record['issuer'], (string) ($record['document_number'] ?? ''), (string) ($record['document_date'] ?? ''),
|
|
(string) $record['language'], (string) ($record['version_label'] ?? $record['title']),
|
|
]))));
|
|
}
|
|
}
|