"""Release-gate bypass regressions. Synthetic evidence is confined to temp dirs.""" from pathlib import Path import copy import hashlib import json import shutil import subprocess import sys import tempfile import unittest ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tools')) import release_check as gate from native_report import analyze from source_manifest import addon_manifest from native_provenance import snapshot, bindings, persist, seal_run, archive_source, qualify_archive, MANIFEST, METADATA, PROVENANCE class ReleaseAuditTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory(prefix='erri-release-test-') self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) self.write('addons/core/addon.gproj', 'Project {}') self.write('addons/core/Scripts/Game/Test.c', 'class Test {}') self.write('addons/core/Assets/test.png', b'nonempty resource bytes') self.write('docs/evidence/current/manual.txt', 'Synthetic test fixture, not real acceptance.\n') self.log = 'docs/evidence/current/console.log.txt' self.report = 'docs/evidence/current/report.json' self.manifest = 'docs/evidence/current/source-sha256.json' self.write(self.log, 'ERR SMOKE PASS: synthetic\nERR SMOKE COMPLETE: synthetic\n') self.refresh_native() ordinary = {'status': 'passed', 'evidence': ['docs/evidence/current/manual.txt']} native = {'status': 'passed', 'evidence': [self.report, self.manifest], 'source_manifest': self.manifest, 'native_report': self.report} self.ledger = {'target': 'Everon, vanilla factions, 12 players', 'cases': {name: copy.deepcopy(ordinary) for name in gate.REQUIRED_CASES}, 'development_checks': {'clean_native_runtime': copy.deepcopy(native)}} self.ledger['cases']['independent_native_build'] = copy.deepcopy(native) def write(self, relative, value): path = self.root / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(value if isinstance(value, bytes) else value.encode()) def json(self, relative, value): self.write(relative, json.dumps(value)) def refresh_native(self): captured = snapshot(self.root, ['synthetic-runner-not-an-engine']) archive = Path(self.manifest).parent self.write(self.manifest, captured[0]) self.write(str(archive / MANIFEST), captured[0]) self.write(str(archive / METADATA), captured[1]) self.json(str(archive / PROVENANCE), {'schema_version': 1, 'kind': 'launch_snapshot'}) self.json(str(archive / 'runner.json.txt'), {**bindings(captured), 'started_unix': json.loads(captured[1])['captured_unix'] + 1, 'result': 'smoke-completed', 'rounds_required': 1, 'stopped_unix': json.loads(captured[1])['captured_unix'] + 2, 'source_unchanged_after_run': True, 'end_source_sha256': hashlib.sha256(captured[0]).hexdigest(), 'console_sha256': hashlib.sha256((self.root / self.log).read_bytes()).hexdigest()}) self.json(self.report, analyze((self.root / self.log).read_text(), self.log)) def run_audit(self): self.json('docs/acceptance.json', self.ledger) return gate.audit(self.root) def blocked(self, fragment): errors = self.run_audit() self.assertTrue(errors, 'malformed evidence incorrectly qualified') self.assertTrue(any(fragment in error for error in errors), errors) def test_complete_synthetic_fixture_qualifies(self): self.assertEqual(self.run_audit(), []) def test_failed_clean_runtime_blocks_even_when_all_cases_pass(self): self.ledger['development_checks']['clean_native_runtime']['status'] = 'failed' self.blocked('clean_native_runtime: status is failed') def test_every_required_case_is_mandatory(self): for name in gate.REQUIRED_CASES: with self.subTest(name=name): saved = self.ledger['cases'].pop(name) self.blocked(name + ': required first-release case is missing') self.ledger['cases'][name] = saved def test_unknown_case_rejected(self): self.ledger['cases']['fake_replacement'] = {'status': 'passed', 'evidence': [self.log]} self.blocked('unknown first-release case') def test_missing_runtime_gate_rejected(self): self.ledger['development_checks'].clear() self.blocked('required clean-runtime gate is missing') def test_unknown_and_malformed_statuses_rejected_everywhere(self): entry = self.ledger['cases']['escape_loop'] for status in ('PASS', 'waived', '', None, True, {}, []): with self.subTest(status=status): entry['status'] = status self.blocked('unknown or malformed status') entry['status'] = 'passed' self.ledger['development_checks']['historical'] = {'status': 'waived', 'evidence': []} self.blocked('unknown or malformed status') def test_malformed_sections_entries_and_evidence(self): original = copy.deepcopy(self.ledger) for section in ('cases', 'development_checks'): self.ledger = copy.deepcopy(original) self.ledger[section] = [] self.blocked(section + ' must be an object') for entry in (None, [], 'passed'): self.ledger = copy.deepcopy(original) self.ledger['cases']['escape_loop'] = entry self.blocked('entry must be an object') for evidence in (None, self.log, {}, [None], [42]): self.ledger = copy.deepcopy(original) self.ledger['cases']['escape_loop']['evidence'] = evidence self.blocked('evidence') def test_passed_requires_nonempty_evidence(self): self.ledger['cases']['escape_loop']['evidence'] = [] self.blocked('requires nonempty evidence') def test_empty_files_and_whitespace_cannot_qualify(self): for value in (b'', b' \r\n\t'): self.write('docs/evidence/empty.txt', value) self.ledger['cases']['escape_loop']['evidence'] = ['docs/evidence/empty.txt'] self.blocked('evidence is empty') def test_missing_and_directory_evidence_rejected(self): for path in ('docs/evidence/missing.txt', 'docs/evidence/current'): self.ledger['cases']['escape_loop']['evidence'] = [path] self.blocked('not an existing regular file') def test_unsafe_paths_rejected(self): for path in ('/etc/passwd', '../outside', 'docs/../docs/evidence/current/manual.txt', './docs/evidence/current/manual.txt', 'docs//evidence/current/manual.txt', 'C:\\evidence.txt', '.git/config', 'docs/evidence/\x00invalid'): with self.subTest(path=path): self.ledger['cases']['escape_loop']['evidence'] = [path] self.assertTrue(self.run_audit()) def test_symlinks_even_inside_repository_rejected(self): for target in (self.root / self.log, Path('/etc/passwd')): link = self.root / 'docs/evidence/link.txt' link.symlink_to(target) self.ledger['cases']['escape_loop']['evidence'] = ['docs/evidence/link.txt'] self.assertTrue(self.run_audit()) link.unlink() def test_symlink_loop_returns_error_without_crash(self): link = self.root / 'docs/evidence/loop' link.symlink_to(link) self.ledger['cases']['escape_loop']['evidence'] = ['docs/evidence/loop'] self.assertTrue(self.run_audit()) def test_duplicate_paths_rejected(self): self.ledger['cases']['escape_loop']['evidence'] = [self.log, self.log] self.blocked('duplicate evidence path') def test_missing_explicit_native_fields_rejected(self): for field in ('source_manifest', 'native_report'): value = self.ledger['cases']['independent_native_build'].pop(field) self.blocked('independent_native_build.' + field) self.ledger['cases']['independent_native_build'][field] = value def test_explicit_native_fields_must_be_listed_as_evidence(self): self.ledger['cases']['independent_native_build']['evidence'] = [self.log] self.blocked('must also appear') def test_current_native_qualification_rejects_changed_added_deleted_files(self): path = self.root / 'addons/core/Scripts/Game/Test.c' original = path.read_bytes() path.write_text('class Changed {}') self.blocked('changed=1') path.write_bytes(original) added = self.root / 'addons/core/Assets/new.custom-resource' added.write_bytes(b'resource') self.blocked('missing=1') added.unlink() path.unlink() self.blocked('extra=1') def test_clean_runtime_also_requires_current_manifest(self): self.ledger['cases']['independent_native_build']['status'] = 'pending' self.write('addons/core/Scripts/Game/Test.c', 'class Changed {}') self.blocked('clean_native_runtime.source_manifest: manifest does not match') def test_incomplete_and_malformed_manifest_rejected(self): for manifest in ({}, [], {'addons/core/addon.gproj': 'not-sha256'}): self.json(self.manifest, manifest) self.blocked('manifest must be a nonempty') self.json(self.manifest, {'addons/core/addon.gproj': '0' * 64}) self.blocked('manifest does not match') def test_native_report_cannot_hide_raw_errors(self): self.write(self.log, 'ERR SMOKE PASS: synthetic\nWORLD (E): retained error\nERR SMOKE COMPLETE: synthetic\n') self.blocked('does not match analysis') self.refresh_native() self.blocked('no script, engine or end-screen errors') def test_each_native_failure_category_blocks(self): for failure in ('ERR SMOKE FAIL: assertion', 'Virtual Machine Exception', 'SCRIPT (E): exception', 'ANIMATION (E): unknown resource', 'ENGINE (F): Crashed', 'Could not find expected endscreen', "entity won't take damage and wont replicate"): self.write(self.log, 'ERR SMOKE PASS: test\n' + failure + '\nERR SMOKE COMPLETE: test\n') self.refresh_native() self.blocked('native raw log must contain') def test_truncated_and_assertion_free_logs_rejected(self): for text in ('ERR SMOKE PASS: test\n', 'ERR SMOKE COMPLETE: test\n', 'unrelated log\n'): self.write(self.log, text) self.refresh_native() self.blocked('successful completion/assertions') def test_native_report_types_and_missing_fields_rejected(self): report = analyze((self.root / self.log).read_text(), self.log) for bad in ({}, [], dict(report, smoke_completions=True), dict(report, engine_errors='')): self.json(self.report, bad) self.blocked('native_report') def test_native_artifacts_cannot_be_mixed_between_archives(self): self.write('docs/evidence/other/source-sha256.json', (self.root / self.manifest).read_bytes()) entry = self.ledger['cases']['independent_native_build'] entry['source_manifest'] = 'docs/evidence/other/source-sha256.json' entry['evidence'].append(entry['source_manifest']) self.blocked('same evidence archive') def test_historical_development_manifests_need_not_match_current(self): self.json('docs/evidence/historical/source-sha256.json', {'old/path.c': '0' * 64}) self.ledger['development_checks']['historical'] = { 'status': 'passed', 'evidence': ['docs/evidence/historical/source-sha256.json']} self.ledger['development_checks']['expected_fault'] = {'status': 'failed', 'evidence': []} self.assertEqual(self.run_audit(), []) def test_generated_state_excluded_but_new_resource_formats_included(self): self.write('addons/core/resourceDatabase.rdb', 'generated') self.write('addons/core/resources.pak', 'generated') self.write('addons/core/profiles/runtime.json', 'generated') self.assertEqual(self.run_audit(), []) self.write('addons/core/Scripts/Game/resourceDatabase.rdb_fake.c', 'class New {}') self.blocked('missing=1') def test_source_symlink_cannot_hide_from_manifest(self): (self.root / 'addons/core/Scripts/Game/Alias.c').symlink_to(self.root / 'addons/core/Scripts/Game/Test.c') self.blocked('source/resource symlink is not supported') def test_duplicate_json_keys_and_invalid_json_fail_closed(self): for raw in ('{"cases":{},"cases":{}}', '{', '[]', '{"target":NaN}'): self.write('docs/acceptance.json', raw) self.assertTrue(gate.audit(self.root)) def test_symlinked_ledger_rejected(self): self.json('docs/real-ledger.json', self.ledger) (self.root / 'docs/acceptance.json').symlink_to(self.root / 'docs/real-ledger.json') self.assertTrue(gate.audit(self.root)) def test_checker_is_read_only(self): self.json('docs/acceptance.json', self.ledger) before = {str(path.relative_to(self.root)): path.read_bytes() for path in self.root.rglob('*') if path.is_file()} self.assertEqual(gate.audit(self.root), []) after = {str(path.relative_to(self.root)): path.read_bytes() for path in self.root.rglob('*') if path.is_file()} self.assertEqual(after, before) def test_archive_command_uses_shared_manifest_and_report(self): tools = self.root / 'tools' tools.mkdir() for name in ('archive_native.py', 'native_report.py', 'source_manifest.py', 'native_provenance.py'): shutil.copyfile(ROOT / 'tools' / name, tools / name) session = self.root / 'synthetic-session' session.mkdir() (session / 'console.log').write_text((self.root / self.log).read_text()) archive = self.root / Path(self.manifest).parent for name in (MANIFEST, METADATA): shutil.copyfile(archive / name, session / name) shutil.copyfile(archive / 'runner.json.txt', session / 'runner.json') subprocess.run([sys.executable, str(tools / 'archive_native.py'), str(session), 'new-archive'], capture_output=True, text=True, check=True) archive = self.root / 'docs/evidence/new-archive' self.assertEqual(json.loads((archive / 'source-sha256.json').read_text()), addon_manifest(self.root)) report = json.loads((archive / 'report.json').read_text()) self.assertEqual(report, analyze((self.root / report['log']).read_text(), report['log'])) def test_native_report_command_compatibility(self): result = subprocess.run([sys.executable, str(ROOT / 'tools/native_report.py'), str(self.root / self.log)], capture_output=True, text=True, check=True) self.assertEqual(json.loads(result.stdout), analyze((self.root / self.log).read_text(), self.root / self.log)) def test_archive_time_provenance_cannot_qualify_native_gates(self): self.json('docs/evidence/current/' + PROVENANCE, {'schema_version': 1, 'kind': 'archive_time_only'}) self.blocked('requires launch_snapshot provenance') def test_missing_launch_snapshot_is_not_current_evidence(self): (self.root / 'docs/evidence/current' / METADATA).unlink() self.blocked('launch_provenance') def test_modified_launch_bytes_and_runner_binding_rejected(self): path = self.root / 'docs/evidence/current' / MANIFEST path.write_bytes(path.read_bytes() + b'\n') self.blocked('launch manifest byte hash') self.refresh_native() runner = self.root / 'docs/evidence/current/runner.json.txt' data = json.loads(runner.read_text()) data['launch_id'] = '00000000-0000-0000-0000-000000000000' runner.write_text(json.dumps(data)) self.blocked('runner launch binding') def test_snapshot_after_process_start_rejected(self): runner = self.root / 'docs/evidence/current/runner.json.txt' data = json.loads(runner.read_text()) data['started_unix'] = 1 runner.write_text(json.dumps(data)) self.blocked('snapshot must precede process launch') def test_runner_failure_and_incomplete_repeated_rounds_cannot_qualify(self): runner = self.root / 'docs/evidence/current/runner.json.txt' data = json.loads(runner.read_text()) data['result'] = 'process-exited' runner.write_text(json.dumps(data)) self.blocked('completed runner session') data['result'] = 'smoke-completed' data['rounds_required'] = 2 runner.write_text(json.dumps(data)) self.blocked('successful completion/assertions') def test_prelaunch_snapshot_remains_old_when_persisted_after_edit(self): captured = snapshot(self.root, ['synthetic']) self.write('addons/core/Scripts/Game/Test.c', 'class ChangedAfterLaunch {}') session = self.root / 'synthetic-late-session' session.mkdir() persist(session, captured) self.assertEqual((session / MANIFEST).read_bytes(), captured[0]) (session / 'console.log').write_text('Synthetic old run') self.json('synthetic-late-session/runner.json', {**bindings(captured), 'started_unix': json.loads(captured[1])['captured_unix'] + 1, 'stopped_unix': json.loads(captured[1])['captured_unix'] + 2, 'source_unchanged_after_run': True, 'end_source_sha256': hashlib.sha256(captured[0]).hexdigest(), 'console_sha256': hashlib.sha256((session / 'console.log').read_bytes()).hexdigest()}) with self.assertRaisesRegex(ValueError, 'differ from pre-launch snapshot'): archive_source(session, self.root) def test_legacy_archive_requires_opt_in_and_is_marked_historical(self): session = self.root / 'synthetic-legacy' session.mkdir() with self.assertRaisesRegex(ValueError, 'no pre-launch source snapshot'): archive_source(session, self.root) manifest, provenance, launched = archive_source(session, self.root, allow_legacy=True) self.assertFalse(launched) self.assertEqual(provenance['kind'], 'archive_time_only') self.assertEqual(provenance['qualification'], 'historical_only') self.assertEqual(json.loads(manifest), addon_manifest(self.root)) def test_partial_launch_evidence_cannot_fall_back_to_legacy(self): session = self.root / 'synthetic-partial' session.mkdir() (session / METADATA).write_text('{}') with self.assertRaisesRegex(ValueError, 'missing or symlinked launch evidence'): archive_source(session, self.root, allow_legacy=True) def test_source_manifest_copy_must_match_exact_launch_bytes(self): path = self.root / self.manifest path.write_bytes(path.read_bytes() + b'\n') self.blocked('differs from exact launch snapshot bytes') def test_recomputed_report_cannot_relabel_a_different_old_raw_log(self): self.write(self.log, 'ERR SMOKE PASS: different synthetic run\nERR SMOKE COMPLETE: other\n') self.json(self.report, analyze((self.root / self.log).read_text(), self.log)) self.blocked('raw native log differs from stopped runner log hash') def test_report_cannot_use_an_unsealed_neighbor_log(self): other = 'docs/evidence/current/other.log.txt' self.write(other, 'ERR SMOKE PASS: unrelated\nERR SMOKE COMPLETE: unrelated\n') self.json(self.report, analyze((self.root / other).read_text(), other)) self.blocked('exact console.log.txt sealed by the stopped runner') def test_stopped_source_change_blocks_even_if_current_files_match_launch(self): runner = self.root / 'docs/evidence/current/runner.json.txt' data = json.loads(runner.read_text()) data['source_unchanged_after_run'] = False data['end_source_sha256'] = '0' * 64 runner.write_text(json.dumps(data)) self.blocked('changed between launch and stopped-run snapshot') def test_seal_uses_stopped_sources_and_exact_log_bytes(self): session = self.root / 'synthetic-seal' session.mkdir() captured = snapshot(self.root, ['synthetic']) (session / 'console.log').write_bytes(b'exact synthetic log\r\n') sealed = seal_run(self.root, session, captured) self.assertTrue(sealed['source_unchanged_after_run']) self.assertEqual(sealed['console_sha256'], hashlib.sha256(b'exact synthetic log\r\n').hexdigest()) self.write('addons/core/Scripts/Game/Test.c', 'class ChangedDuringRun {}') self.assertFalse(seal_run(self.root, session, captured)['source_unchanged_after_run']) def test_archive_cli_requires_explicit_legacy_import(self): tools = self.root / 'tools' tools.mkdir() for name in ('archive_native.py', 'native_report.py', 'source_manifest.py', 'native_provenance.py'): shutil.copyfile(ROOT / 'tools' / name, tools / name) session = self.root / 'synthetic-legacy-command' session.mkdir() (session / 'console.log').write_text((self.root / self.log).read_text()) command = [sys.executable, str(tools / 'archive_native.py'), str(session), 'legacy-archive'] rejected = subprocess.run(command, capture_output=True, text=True) self.assertEqual(rejected.returncode, 2) self.assertFalse((self.root / 'docs/evidence/legacy-archive').exists()) subprocess.run(command + ['--allow-legacy-archive-time-source'], capture_output=True, text=True, check=True) provenance = json.loads((self.root / 'docs/evidence/legacy-archive' / PROVENANCE).read_text()) self.assertEqual(provenance['kind'], 'archive_time_only') with self.assertRaisesRegex(ValueError, 'requires launch_snapshot provenance'): qualify_archive(self.root / 'docs/evidence/legacy-archive', addon_manifest(self.root)) if __name__ == '__main__': unittest.main()