argument('manifest')); $stagingReal = realpath($staging); if (! $manifestPath || ! $stagingReal || ! str_starts_with(strtolower($manifestPath), strtolower($stagingReal.DIRECTORY_SEPARATOR)) || ! is_file($manifestPath)) { $this->error('Manifest is missing or outside the private document-import staging directory.'); return self::FAILURE; } foreach ([ 'documents' => ['id', 'source_system', 'external_id', 'version_no', 'data_classification', 'file_name', 'media_type', 'storage_key', 'sha256', 'status', 'size_bytes', 'issuer', 'issued_on', 'uploaded_by', 'rights_status', 'original_source', 'malware_scan_status'], 'portal_resources' => ['document_id', 'title', 'category', 'topic', 'published', 'source_reference', 'approved_by', 'description', 'document_type', 'document_number', 'document_date', 'source_url', 'language', 'version_label', 'source_confirmation', 'legal_status', 'rights_status', 'document_identity_key', 'metadata_version', 'created_by'], ] as $table => $columns) { if (! Schema::hasColumns($table, $columns)) { $this->error('Document-library schema is incomplete; run the documented schema compatibility check before importing.'); return self::FAILURE; } } try { $manifest = json_decode((string) file_get_contents($manifestPath), true, 512, JSON_THROW_ON_ERROR); } catch (Throwable) { $this->error('Manifest is not valid JSON.'); return self::FAILURE; } if (! is_array($manifest) || ! isset($manifest['documents']) || ! is_array($manifest['documents']) || count($manifest['documents']) > 100) { $this->error('Manifest must contain at most 100 documents in a documents array.'); return self::FAILURE; } $apply = (bool) $this->option('apply'); $actorId = $this->option('actor'); if ($apply && (! is_numeric($actorId) || ! DB::table('users')->where('id', (int) $actorId)->exists())) { $this->error('Apply requires --actor=; no author identity will be fabricated.'); return self::FAILURE; } $disk = Storage::disk('local'); $seenHashes = []; $seenExternalIds = []; $dispatch = []; $failures = 0; foreach ($manifest['documents'] as $index => $record) { $label = is_array($record) ? (string) ($record['external_id'] ?? '#'.($index + 1)) : '#'.($index + 1); try { $validated = $this->validateRecord($record, $stagingReal); $hash = $validated['sha256']; if (isset($seenHashes[$hash])) { $this->line($label.' | duplicate bytes within batch; references item '.$seenHashes[$hash]); continue; } $seenHashes[$hash] = $label; if (isset($seenExternalIds[$validated['external_id']])) { $this->warn($label.' | external_id already occurs in this batch with different bytes; identity review required'); continue; } $seenExternalIds[$validated['external_id']] = $hash; $existing = DB::table('documents')->where('sha256', $hash)->whereNull('deleted_at')->first(); if ($existing) { $this->line($label.' | already imported by SHA-256; document='.$existing->id.'; no duplicate'); continue; } $externalMatch = DB::table('documents')->where('source_system', 'document-batch-import') ->where('external_id', $validated['external_id'])->whereNull('deleted_at')->first(); if ($externalMatch) { $this->warn($label.' | external_id already belongs to document '.$externalMatch->id.' with different bytes; version review required'); continue; } $identity = $this->identity($validated); $identityMatch = DB::table('portal_resources')->where('document_identity_key', $identity)->first(); if ($identityMatch) { $this->warn($label.' | identity matches resource '.$identityMatch->id.' but bytes differ; version review required'); continue; } $this->line($label.' | '.($apply ? 'will import as private draft' : 'preview: private draft')." | bytes={$validated['size_bytes']} | sha256={$hash}"); if (! $apply) { continue; } $documentId = (string) Str::uuid(); $storageKey = 'portal-documents/'.$documentId; $source = fopen($validated['source_path'], 'rb'); if (! is_resource($source)) { throw new RuntimeException('Source could not be opened.'); } try { $stored = $disk->put($storageKey, $source); } finally { fclose($source); } $storedPath = $disk->path($storageKey); $storedHash = $stored && is_file($storedPath) ? hash_file('sha256', $storedPath) : false; if (! $stored || ! is_string($storedHash) || ! hash_equals($hash, $storedHash)) { $disk->delete($storageKey); throw new RuntimeException('Private copy failed the checksum verification.'); } try { $resourceId = DB::transaction(function () use ($validated, $documentId, $storageKey, $hash, $identity, $actorId): int { DB::select('SELECT pg_advisory_xact_lock(hashtext(?))', ['document-batch:'.$hash]); DB::select('SELECT pg_advisory_xact_lock(hashtext(?))', ['document-batch-identity:'.$identity]); if (DB::table('documents')->where('sha256', $hash)->whereNull('deleted_at')->exists()) { throw new RuntimeException('Matching bytes were registered concurrently; rerun preview.'); } if (DB::table('portal_resources')->where('document_identity_key', $identity)->exists()) { throw new RuntimeException('A matching document identity was registered concurrently; version review required.'); } $now = now(); DB::table('documents')->insert([ 'id' => $documentId, 'source_system' => 'document-batch-import', 'external_id' => $validated['external_id'], 'version_no' => 1, 'data_classification' => 'internal', 'file_name' => $validated['file_name'], 'media_type' => $validated['media_type'], 'storage_key' => $storageKey, 'sha256' => $hash, 'status' => 'quarantined', 'size_bytes' => $validated['size_bytes'], 'issuer' => $validated['issuer'], 'issued_on' => $validated['document_date'], 'uploaded_by' => (int) $actorId, 'rights_status' => 'unconfirmed', 'original_source' => $validated['source_url'], 'malware_scan_status' => 'pending', 'created_at' => $now, 'updated_at' => $now, ]); $resourceId = DB::table('portal_resources')->insertGetId([ 'document_id' => $documentId, 'title' => $validated['title'], 'category' => $validated['category'], 'topic' => $validated['topic'], 'legislation' => $validated['legislation'], 'essential' => false, 'published' => false, 'source_reference' => $validated['source_reference'], 'approved_by' => null, 'description' => $validated['description'], 'document_type' => $validated['document_type'], 'document_number' => $validated['document_number'], 'document_date' => $validated['document_date'], 'source_url' => $validated['source_url'], 'language' => $validated['language'], 'version_label' => $validated['version_label'], 'territorial_scope' => 'unknown', 'source_confirmation' => 'unconfirmed', 'legal_status' => 'unknown', 'rights_status' => 'unconfirmed', 'document_identity_key' => $identity, 'metadata_version' => 1, 'created_by' => (int) $actorId, 'last_reviewed_by' => null, 'independent_review' => false, 'created_at' => $now, 'updated_at' => $now, ]); $resource = DB::table('portal_resources')->where('id', $resourceId)->first(); DB::table('portal_resource_revisions')->insert([ 'resource_id' => $resourceId, 'revision_no' => 1, 'change_type' => 'created', 'snapshot' => json_encode(['resource' => $resource, 'associations' => []], JSON_THROW_ON_ERROR), 'actor_user_id' => (int) $actorId, 'created_at' => $now, ]); DB::table('audit_events')->insert([ 'id' => (string) Str::uuid(), 'source_system' => 'document-batch-import', 'external_id' => $validated['external_id'], 'data_classification' => 'internal', 'actor_user_id' => (int) $actorId, 'entity_type' => 'document', 'entity_id' => $documentId, 'action' => 'private_document_batch_imported', 'after_data' => json_encode(['sha256' => $hash, 'resource_id' => $resourceId, 'published' => false], JSON_THROW_ON_ERROR), 'occurred_at' => $now, 'created_at' => $now, ]); return $resourceId; }); $dispatch[] = $documentId; $this->info($label.' | private draft created | resource='.$resourceId.'; pending scan; not published'); } catch (Throwable $exception) { $disk->delete($storageKey); throw $exception; } } catch (Throwable $exception) { $this->error($label.' | not imported: '.$exception->getMessage()); $failures++; } } foreach ($dispatch as $documentId) { ScanDocumentForMalware::dispatch($documentId)->afterCommit(); } $this->line($apply ? 'New scans queued: '.count($dispatch).'.' : 'Preview only; database and storage unchanged.'); return $failures > 0 ? self::FAILURE : self::SUCCESS; } /** @return array */ private function validateRecord(mixed $record, string $stagingRoot): array { if (! is_array($record)) { throw new RuntimeException('Each documents entry must be an object.'); } foreach (['external_id', 'relative_path', 'title', 'issuer'] as $required) { if (! isset($record[$required]) || ! is_string($record[$required]) || trim($record[$required]) === '') { throw new RuntimeException('Required manifest field is missing: '.$required.'.'); } } if (strlen($record['external_id']) > 120 || strlen($record['title']) > 250 || strlen($record['issuer']) > 180) { throw new RuntimeException('A metadata value exceeds its permitted length.'); } if (isset($record['file_name']) && (! is_string($record['file_name']) || strlen(basename($record['file_name'])) > 255)) { throw new RuntimeException('File name exceeds its permitted length.'); } if (isset($record['document_number']) && (! is_string($record['document_number']) || strlen($record['document_number']) > 120)) { throw new RuntimeException('Document number exceeds its permitted length.'); } if (isset($record['document_date'])) { if (! is_string($record['document_date']) || ! preg_match('/^(\d{4})-(\d{2})-(\d{2})$/', $record['document_date'], $dateParts) || ! checkdate((int) $dateParts[2], (int) $dateParts[3], (int) $dateParts[1])) { throw new RuntimeException('Document date must use a valid YYYY-MM-DD value.'); } } if (isset($record['document_type']) && ! in_array($record['document_type'], ['legislation', 'procedure', 'official_form', 'digital_adaptation', 'guidance_template', 'guide', 'study', 'reference_data', 'unknown'], true)) { throw new RuntimeException('Document type is not supported by the catalogue.'); } $path = realpath($stagingRoot.DIRECTORY_SEPARATOR.$record['relative_path']); if (! $path || ! str_starts_with(strtolower($path), strtolower($stagingRoot.DIRECTORY_SEPARATOR)) || ! is_file($path)) { throw new RuntimeException('Source file is missing or outside the private staging directory.'); } $size = filesize($path); if (! is_int($size) || $size < 1 || $size > self::MAX_BYTES) { throw new RuntimeException('File size is outside the 1 byte to 10 MiB import limit.'); } $mime = (new \finfo(FILEINFO_MIME_TYPE))->file($path); $allowed = ['application/pdf', 'text/plain', 'text/csv', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'image/png', 'image/jpeg', 'image/webp']; if (! is_string($mime) || ! in_array($mime, $allowed, true)) { throw new RuntimeException('Detected media type is not supported by the document library.'); } $sha = hash_file('sha256', $path); if (! is_string($sha)) { throw new RuntimeException('File hash could not be calculated.'); } $sourceUrl = $record['source_url'] ?? null; if ($sourceUrl !== null && (! is_string($sourceUrl) || strlen($sourceUrl) > 2000 || ! filter_var($sourceUrl, FILTER_VALIDATE_URL) || strtolower((string) parse_url($sourceUrl, PHP_URL_SCHEME)) !== 'https')) { throw new RuntimeException('Source URL, when supplied, must be a valid HTTPS URL.'); } return [ ...$record, 'source_path' => $path, 'file_name' => basename((string) ($record['file_name'] ?? $path)), 'size_bytes' => $size, 'media_type' => $mime, 'sha256' => $sha, 'category' => (string) ($record['category'] ?? 'general'), 'topic' => (string) ($record['topic'] ?? 'unclassified'), 'legislation' => (bool) ($record['legislation'] ?? false), 'document_type' => (string) ($record['document_type'] ?? 'unknown'), 'document_number' => $record['document_number'] ?? null, 'document_date' => $record['document_date'] ?? null, 'language' => (string) ($record['language'] ?? 'unknown'), 'version_label' => $record['version_label'] ?? null, 'source_url' => $record['source_url'] ?? null, 'source_reference' => $record['source_reference'] ?? 'Origem por completar.', 'description' => $record['description'] ?? 'Documento importado para revisão documental.', ]; } /** @param array $record */ private function identity(array $record): string { return hash('sha256', mb_strtolower(implode('|', array_map('trim', [ (string) $record['issuer'], (string) ($record['document_number'] ?? ''), (string) ($record['document_date'] ?? ''), (string) $record['language'], (string) ($record['version_label'] ?? $record['title']), ])))); } }