78 lines
4.9 KiB
PHP
78 lines
4.9 KiB
PHP
<?php
|
|
|
|
namespace App\Console\Commands;
|
|
|
|
use App\Services\Ai\EvidenceRepository;
|
|
use Illuminate\Console\Command;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Str;
|
|
|
|
final class AiBenchmark extends Command
|
|
{
|
|
protected $signature = 'ai:benchmark';
|
|
|
|
protected $description = 'Mede recuperação lexical PT/EN com dados sintéticos e rollback numa base *_test';
|
|
|
|
public function handle(EvidenceRepository $evidence): int
|
|
{
|
|
if (! app()->environment('testing') || DB::connection()->getDriverName() !== 'pgsql'
|
|
|| ! str_ends_with(DB::connection()->getDatabaseName(), '_test')
|
|
|| DB::connection()->getDatabaseName() !== DB::selectOne('select current_database() as name')->name) {
|
|
$this->error('Apenas numa base PostgreSQL isolada *_test, com APP_ENV=testing.');
|
|
|
|
return self::FAILURE;
|
|
}
|
|
$cases = [
|
|
['pt', 'formação técnica', 'Exemplo sintético: formação técnica de electricistas em oficinas.', true],
|
|
['pt', 'cadeia logística', 'Exemplo sintético: a cadeia logística inclui armazenamento e transporte.', true],
|
|
['pt', 'energia solar', 'Exemplo sintético: manutenção de equipamentos de energia solar.', true],
|
|
['pt', 'taxa oficial', 'Exemplo sintético sem informação legal.', false],
|
|
['en', 'vocational training', 'Synthetic example: vocational training for equipment maintenance.', true],
|
|
['en', 'logistics corridor', 'Synthetic example: a logistics corridor connects warehouses and transport hubs.', true],
|
|
['en', 'solar energy', 'Synthetic example: solar energy equipment requires maintenance.', true],
|
|
['en', 'official tax', 'Synthetic example with no legal information.', false],
|
|
];
|
|
DB::beginTransaction();
|
|
try {
|
|
// Unique synthetic fixtures are visible only within this transaction.
|
|
$author = DB::table('users')->insertGetId(['name' => 'Synthetic benchmark author', 'email' => Str::uuid().'@example.invalid', 'password' => 'unused']);
|
|
$reviewer = DB::table('users')->insertGetId(['name' => 'Synthetic benchmark reviewer', 'email' => Str::uuid().'@example.invalid', 'password' => 'unused']);
|
|
$measurements = [];
|
|
$passed = 0;
|
|
foreach ($cases as [$language, $query, $text, $expected]) {
|
|
$document = (string) Str::uuid();
|
|
$source = (string) Str::uuid();
|
|
DB::table('documents')->insert(['id' => $document, 'file_name' => 'synthetic.txt', 'media_type' => 'text/plain',
|
|
'storage_key' => 'synthetic-'.$document, 'sha256' => hash('sha256', $text), 'status' => 'active', 'data_classification' => 'internal']);
|
|
DB::table('ai_sources')->insert(['id' => $source, 'document_id' => $document, 'document_version' => 1,
|
|
'document_checksum' => hash('sha256', $text), 'title' => 'Synthetic benchmark', 'language' => $language,
|
|
'nature' => 'illustrative', 'source_url' => 'https://example.org/synthetic', 'source_owner' => 'Test fixture',
|
|
'evidence_reference' => 'Synthetic benchmark only', 'source_date' => now()->toDateString(),
|
|
'valid_until' => now()->addDay()->toDateString(), 'status' => 'approved', 'submitted_by' => $author,
|
|
'reviewed_by' => $reviewer, 'reviewed_at' => now()]);
|
|
DB::table('ai_chunks')->insert(['id' => (string) Str::uuid(), 'source_id' => $source, 'position' => 0,
|
|
'language' => $language, 'content' => $text, 'checksum' => hash('sha256', $text)]);
|
|
$latencies = [];
|
|
for ($i = 0; $i < 5; $i++) {
|
|
$start = hrtime(true);
|
|
$results = $evidence->search($query, $language, ['source_id' => $source]);
|
|
$latencies[] = round((hrtime(true) - $start) / 1000000, 3);
|
|
}
|
|
$ok = ($results !== []) === $expected;
|
|
$passed += $ok ? 1 : 0;
|
|
sort($latencies);
|
|
$measurements[] = ['language' => $language, 'query' => $query, 'expected_match' => $expected,
|
|
'passed' => $ok, 'p50_ms' => $latencies[2], 'max_ms' => $latencies[4]];
|
|
}
|
|
$this->line(json_encode(['executed_at' => now()->toIso8601String(), 'backend' => 'PostgreSQL FTS', 'model' => null,
|
|
'synthetic' => true, 'passed' => $passed, 'cases' => count($cases), 'repetitions_per_case' => 5,
|
|
'measurements' => $measurements, 'generative_benchmark' => 'not_executed_no_runtime_or_model',
|
|
'limitations' => ['small_single_source_cases', 'not_a_representative_benchmark', 'not_human_validated_PT_MZ', 'no_concurrency_or_load_test']], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR));
|
|
|
|
return $passed === count($cases) ? self::SUCCESS : self::FAILURE;
|
|
} finally {
|
|
DB::rollBack();
|
|
}
|
|
}
|
|
}
|