#!/usr/bin/env python3 """Gitea -> isolated verification/build -> immutable release -> atomic switch. Runs as the existing deployment OS user. Never resets the served checkout, copies a database, runs production migrations, imports content or changes grants. The installed copy and /etc configuration are controlled by the server operator. """ from __future__ import annotations import argparse, base64, contextlib, datetime, hashlib, io, json, os, re import shutil, socket, subprocess, tarfile, time, urllib.request, uuid from pathlib import Path SHA = re.compile(r'^[0-9a-f]{40}$') def digest(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def write_json(path: Path, value: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name('.'+path.name+'.'+uuid.uuid4().hex) temporary.write_text(json.dumps(value, indent=2)+'\n', encoding='utf-8') os.chmod(temporary, 0o640) os.replace(temporary, path) def source_contract(root: Path) -> dict[str, str]: return {p.relative_to(root).as_posix(): hashlib.sha256(p.read_bytes().replace(b'\r\n', b'\n')).hexdigest() for p in sorted((root/'database/migrations').glob('*.php'))} def tree_hashes(root: Path) -> dict[str, str]: result = {} for directory, folders, files in os.walk(root, followlinks=False): parent = Path(directory) folders[:] = [name for name in folders if not (parent/name).is_symlink() and name not in ['.git', 'node_modules'] and (parent/name).relative_to(root).as_posix() not in ['storage', 'bootstrap/cache']] for name in files: path=parent/name;rel=path.relative_to(root).as_posix() if path.is_symlink() or rel=='.env' or rel.startswith('public/build/assets/'):continue result[rel]=digest(path) return result def refuse_drift(root: Path, expected: dict[str, str]) -> None: actual = tree_hashes(root) if actual != expected: changed = sorted(p for p in actual.keys() | expected.keys() if actual.get(p) != expected.get(p)) raise RuntimeError('Source/build drift requires reconciliation: '+', '.join(changed[:12])) def archive_source(data: bytes, root: Path) -> None: with tarfile.open(fileobj=io.BytesIO(data)) as archive: members = archive.getmembers() for item in members: path = (root/item.name).resolve() if not path.is_relative_to(root.resolve()) or not (item.isfile() or item.isdir()): raise RuntimeError('Unsafe source archive entry') if item.name == '.env' or item.name == 'public/hot' or (item.isfile() and item.name.startswith('storage/app/private/') and item.name != 'storage/app/private/.gitignore'): raise RuntimeError('Private/runtime files must not be distributed by Git') # Debian's installed Python predates tarfile extraction filters. Validate # every entry first, then create only ordinary directories/files ourselves. for item in members: path=root/item.name if item.isdir(): path.mkdir(mode=0o750,parents=True,exist_ok=True) else: path.parent.mkdir(mode=0o750,parents=True,exist_ok=True) source=archive.extractfile(item) if source is None:raise RuntimeError('Missing archive source bytes') with source, path.open('xb') as destination:shutil.copyfileobj(source,destination) os.chmod(path,0o644 | (item.mode & 0o111)) def point_current(link: Path, target: Path) -> None: if link.exists() and not link.is_symlink(): raise RuntimeError('Current path is not a symlink; existing directory preserved') temporary = link.with_name('.'+link.name+'.'+uuid.uuid4().hex) temporary.symlink_to(target, target_is_directory=True) os.replace(temporary, link) def promote(link: Path, candidate: Path, health) -> Path | None: previous = link.resolve() if link.is_symlink() else None point_current(link, candidate) try: health() except Exception: if previous is not None: point_current(link, previous) else: link.unlink() raise return previous class Delivery: def __init__(self, config: dict): self.c = config self.state = Path(config['state_root']) self.releases = Path(config['release_root']) self.current = Path(config['current_link']) self.legacy = Path(config['legacy_root']) self.mirror = self.state/'repository.git' self.log = None self.process_env = dict(os.environ) # Do not inherit an application/CI connection accidentally from the calling shell. for key in list(self.process_env): if key.startswith(('DB_', 'APP_', 'MAIL_', 'CACHE_', 'SESSION_', 'QUEUE_', 'VITE_', 'PORTAL_', 'INVESTMOZ_')): self.process_env.pop(key) def run(self, argv, cwd=None, env=None, timeout=900, log_output=True) -> bytes: result = subprocess.run([str(x) for x in argv], cwd=cwd, env=env or self.process_env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout) if self.log and log_output: self.log.write(result.stdout); self.log.flush() if result.returncode: raise RuntimeError('Verification command failed: '+str(argv[0])+' (see protected build log)') return result.stdout def git(self, *args) -> str: return self.run(['git', '--git-dir', self.mirror, *args], timeout=120).decode().strip() def latest(self) -> str: branch = self.c['branch'] self.git('check-ref-format', 'refs/heads/'+branch) self.git('fetch', '--no-tags', 'origin', '+refs/heads/'+branch+':refs/heads/delivery-target') target = self.git('rev-parse', 'refs/heads/delivery-target') if not SHA.fullmatch(target): raise RuntimeError('Invalid fetched revision') return target def ci_env(self) -> dict: flags = json.loads(Path(self.c['ci_environment']).read_text()) # This file contains only reviewed non-secret feature booleans. if not all(re.fullmatch(r'(PORTAL_|INVESTMOZ_)[A-Z0-9_]+', k) and v in ['true', 'false'] for k,v in flags.items()): raise RuntimeError('Unexpected CI feature setting') env = {**self.process_env, **flags} env.update(APP_ENV='testing', APP_DEBUG='false', APP_URL='http://127.0.0.1', APP_KEY='base64:'+base64.b64encode(os.urandom(32)).decode(), APP_NAME='InvesteMoz', DB_CONNECTION='pgsql', DB_URL='', DB_HOST='127.0.0.1', DB_PORT='55441', DB_DATABASE=self.c['test_database'], DB_USERNAME='investmoz_ci', DB_PASSWORD='isolated-ci-only', MAIL_MAILER='array', CACHE_STORE='array', SESSION_DRIVER='array', QUEUE_CONNECTION='sync', INVESTMOZ_HOMOLOGATION_ONLY='false', INVESTMOZ_SCHEMA_TESTS='1', PORTAL_NOTIFICATIONS_ENABLED='false', VITE_APP_NAME='InvesteMoz', NODE_OPTIONS='--max-old-space-size=1536') if self.c['qa_port'] != 55441 or not env['DB_DATABASE'].endswith('_test'): raise RuntimeError('Refuse a non-isolated test target') return env @contextlib.contextmanager def qa_database(self): with socket.socket() as probe: if probe.connect_ex(('127.0.0.1', 55441)) == 0: raise RuntimeError('QA port already occupied; do not reuse or stop another database') name = 'investmoz-ci-'+uuid.uuid4().hex[:16] # Existing Docker; private loopback only, PostgreSQL itself listens on the guarded QA port. cid = self.run(['docker','run','--detach','--name',name,'--label','investmoz.scope=homologacao-ci','--memory=768m','--cpus=1', '--publish','127.0.0.1:55441:55441','--env','POSTGRES_USER=investmoz_ci', '--env','POSTGRES_PASSWORD=isolated-ci-only','--env','POSTGRES_DB='+self.c['test_database'], self.c['postgis_image'],'postgres','-p','55441'], timeout=180).decode().strip().splitlines()[-1] if not re.fullmatch(r'[0-9a-f]{64}', cid): raise RuntimeError('Cannot identify isolated QA container') try: for _ in range(60): ready = subprocess.run(['docker','exec',cid,'pg_isready','-p','55441','-U','investmoz_ci'], stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) if ready.returncode == 0: break time.sleep(1) else: raise RuntimeError('Isolated database did not start') identity = self.run(['docker','exec',cid,'psql','-p','55441','-U','investmoz_ci','-d',self.c['test_database'], '-X','-At','-c','SELECT current_database(), current_setting(\'port\');']).decode().strip() if identity != self.c['test_database']+'|55441': raise RuntimeError('QA identity verification failed') yield finally: # Only the newly created, fully identified CI container and its anonymous volumes. self.run(['docker','rm','--force','--volumes',cid], timeout=60) def build(self, target: str, release: Path, attach_runtime: bool = True) -> None: data = self.run(['git','--git-dir',self.mirror,'archive','--format=tar',target], timeout=120,log_output=False) # Binary archives must never be appended to text build logs. archive_source(data, release) expected = json.loads(Path(self.c['schema_contract']).read_text()) if source_contract(release) != expected: raise RuntimeError('Schema source changed: explicit compatibility/migration decision required; no production SQL executed') env = self.ci_env() bindir = self.state/'bin';bindir.mkdir(exist_ok=True) php = bindir/'php' if not php.exists(): php.symlink_to(self.c['php']) env['PATH'] = str(bindir)+os.pathsep+self.process_env.get('PATH','/usr/bin:/bin') (release/'.env').write_text('APP_ENV=testing\n') (release/'bootstrap/cache').mkdir(parents=True,exist_ok=True) for directory in ['app/private','app/public','framework/cache/data','framework/sessions','framework/views','logs']: (release/'storage'/directory).mkdir(parents=True,exist_ok=True) with self.qa_database(): self.run(['composer','install','--no-interaction','--prefer-dist','--no-scripts'],cwd=release,env=env) self.run([self.c['php'],'artisan','package:discover','--ansi'],cwd=release,env=env) # The guard is verified against the actual PostgreSQL identity, not only environment strings. guard = "require 'vendor/autoload.php'; $app=require 'bootstrap/app.php'; $app->make(Illuminate\\Contracts\\Console\\Kernel::class)->bootstrap(); if (!App\\Support\\IsolatedQaDatabase::allowsSchemaChanges()) exit(91);" self.run([self.c['php'],'-r',guard],cwd=release,env=env) # The project intentionally gates later schemas behind nominal authenticated # installation. Reuse its accepted isolated baseline lifecycle test instead # of bypassing those guards with a generic migrate of every module. self.run([self.c['php'],'vendor/bin/phpunit','--configuration=phpunit.investmoz.xml','--do-not-cache-result'],cwd=release,env=env) self.run(['npm','ci','--no-audit','--no-fund'],cwd=release,env=env) self.run(['npm','run','build'],cwd=release,env=env) self.run(['npm','run','types:check'],cwd=release,env=env) self.run([self.c['php'],'tools/portal-diagnostics-deployment-guard.php'],cwd=release,env=env) # Unit fixtures mock DB/mail and select their own feature settings. # Do not inject the destination's enabled-module flags into those mocks. unit_env={k:v for k,v in env.items() if not k.startswith(('PORTAL_','INVESTMOZ_'))} self.run([self.c['php'],'vendor/bin/phpunit','--configuration=phpunit.xml','--testsuite','Unit','--do-not-cache-result'],cwd=release,env=unit_env) tests=sorted((release/'tests/frontend').glob('*.test.mjs')) self.run(['node','--test',*tests],cwd=release,env=env) # Final production vendor from the same lock, before linking any real configuration. self.run(['composer','install','--no-dev','--no-scripts','--no-interaction','--prefer-dist','--optimize-autoloader'],cwd=release,env=env) self.run(['composer','check-platform-reqs','--no-dev'],cwd=release,env=env) if (release/'public/hot').exists(): raise RuntimeError('Development Vite marker must not ship') manifest=json.loads((release/'public/build/manifest.json').read_text()) for entry in manifest.values(): for path in [entry.get('file'),*entry.get('css',[]),*entry.get('assets',[])]: if path and not (release/'public/build'/path).is_file(): raise RuntimeError('Missing compiled asset') if not attach_runtime: return # Discard only this release's synthetic runtime, retaining it privately outside the served tree. (release/'storage').rename(self.state/'qa-runtime'/release.name) (release/'.env').unlink() for path in (release/'bootstrap/cache').glob('*.php'): path.unlink() (release/'storage').symlink_to(self.legacy/'storage',target_is_directory=True) (release/'.env').symlink_to(self.legacy/'.env') if (release/'public/storage').exists():raise RuntimeError('Unexpected distributed public storage') (release/'public/storage').symlink_to(self.legacy/'storage/app/public',target_is_directory=True) shared=Path(self.c['shared_assets']);shared.mkdir(parents=True,exist_ok=True) assets=release/'public/build/assets' for path in assets.rglob('*'): if not path.is_file():continue dest=shared/path.relative_to(assets);dest.parent.mkdir(parents=True,exist_ok=True) if dest.exists() and digest(dest)!=digest(path):raise RuntimeError('Immutable asset filename collision') if not dest.exists():shutil.copy2(path,dest) # Nginx serves this shared immutable asset pool; old browser bundles remain available. write_json(release/'public/build/revision.json',{'commit':target,'branch':self.c['branch'], 'built_at':datetime.datetime.now(datetime.timezone.utc).isoformat(), 'manifest_sha256':digest(release/'public/build/manifest.json')}) for directory in [release/'bootstrap/cache']: os.chmod(directory,0o2770) def health(self, target: str) -> None: request=urllib.request.Request(self.c['target_url']+'/build/revision.json',headers={'User-Agent':'Mozilla/5.0 InvesteMoz-Delivery','Cache-Control':'no-cache'}) with urllib.request.urlopen(request,timeout=20) as response:info=json.load(response) if info.get('commit')!=target:raise RuntimeError('Served commit differs from built commit') request=urllib.request.Request(self.c['target_url']+'/',headers={'User-Agent':'Mozilla/5.0 InvesteMoz-Delivery','Cache-Control':'no-cache'}) with urllib.request.urlopen(request,timeout=20) as response: if response.status!=200 or response.headers.get('WWW-Authenticate'):raise RuntimeError('Public access regression') def deliver(self, target: str) -> dict: installed_file=self.state/'installed.json' installed=json.loads(installed_file.read_text()) if installed.get('commit')==target: refuse_drift(self.current.resolve(),installed['hashes']) return {'status':'already_installed','commit':target} refuse_drift(self.current.resolve(),installed['hashes']) self.releases.mkdir(parents=True,exist_ok=True) if shutil.disk_usage(self.releases).free < 5 * 1024**3: raise RuntimeError('Insufficient free space for an isolated build; active release preserved') release=self.releases/(target+'-'+uuid.uuid4().hex[:8]);release.mkdir(mode=0o750) self.build(target,release) if self.latest()!=target: latest=self.git('rev-parse','refs/heads/delivery-target') write_json(self.state/'requests'/(latest+'.json'),{'commit':latest,'reason':'newer integration during build'}) return {'status':'superseded','commit':target,'newer_commit':latest} # A programmer may have edited the served release while CI was running. # Recheck immediately before the atomic switch, preserving that work. refuse_drift(self.current.resolve(),installed['hashes']) previous=promote(self.current,release,lambda:self.health(target)) try: self.run([self.c['php'],str(release/'artisan'),'queue:restart'],cwd=release) write_json(self.state/'previous.json',installed) info={'status':'installed','commit':target,'release':str(release),'previous_release':str(previous), 'hashes':tree_hashes(release),'database_migrations':0,'content_imports':0} write_json(installed_file,info) except Exception: if previous is not None:point_current(self.current,previous) raise return {key:value for key,value in info.items() if key!='hashes'} def process(self) -> bool: import fcntl self.state.mkdir(parents=True,exist_ok=True) for directory in ['reports','logs','processed','qa-runtime']: (self.state/directory).mkdir(exist_ok=True) with (self.state/'delivery.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX) succeeded=True for _ in range(3): requests=sorted((self.state/'requests').glob('*.json')) if not requests:return succeeded for path in requests: if not SHA.fullmatch(path.stem):raise RuntimeError('Unexpected queue entry') path.rename(self.state/'processed'/(path.stem+'-'+uuid.uuid4().hex[:8]+'.json')) target=None;report={} with (self.state/'logs'/(str(time.time_ns())+'.log')).open('wb') as self.log: try: target=self.latest();report=self.deliver(target) except Exception as error: succeeded=False report={'status':'failed','commit':target,'error':str(error),'active_release_preserved':True} finally:self.log=None report.update(requested_commits=[p.stem for p in requests],finished_at=datetime.datetime.now(datetime.timezone.utc).isoformat()) write_json(self.state/'reports'/((target or 'fetch-failed')+'-'+str(time.time_ns())+'.json'),report) print(json.dumps(report),flush=True) return succeeded def rollback(self) -> dict: import fcntl with (self.state/'delivery.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX) installed=json.loads((self.state/'installed.json').read_text()) previous=json.loads((self.state/'previous.json').read_text()) refuse_drift(self.current.resolve(),installed['hashes']) prior=Path(previous['release']) refuse_drift(prior,previous['hashes']) current=self.current.resolve() promote(self.current,prior,lambda:self.health(previous['commit'])) try: self.run([self.c['php'],str(prior/'artisan'),'queue:restart'],cwd=prior) write_json(self.state/'installed.json',previous) write_json(self.state/'previous.json',installed) except Exception: point_current(self.current,current) write_json(self.state/'installed.json',installed) raise report={'status':'rolled_back','commit':previous['commit'],'release':str(prior), 'database_restored':False,'shared_data_preserved':True, 'finished_at':datetime.datetime.now(datetime.timezone.utc).isoformat()} write_json(self.state/'reports'/('rollback-'+str(time.time_ns())+'.json'),report) return report def main(): parser=argparse.ArgumentParser();parser.add_argument('--config',default='/etc/investmoz-deploy/delivery.json');parser.add_argument('--rollback',action='store_true');args=parser.parse_args() config=json.loads(Path(args.config).read_text()) if config['target_url']!='https://teste-investidor.tempo.co.mz':raise SystemExit('Exact homologation target required') if Path(config['legacy_root']).resolve()!=Path('/var/www/html/teste-investidor.tempo.co.mz'):raise SystemExit('Unexpected persistent application target') runner=Delivery(config) if args.rollback:print(json.dumps(runner.rollback()),flush=True) else:raise SystemExit(0 if runner.process() else 1) if __name__=='__main__':main()