410 lines
22 KiB
Python
410 lines
22 KiB
Python
"""Release-gate bypass regressions. Synthetic evidence is confined to temp dirs."""
|
|
from pathlib import Path
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tools'))
|
|
import release_check as gate
|
|
from native_report import analyze
|
|
from source_manifest import addon_manifest
|
|
from native_provenance import snapshot, bindings, persist, seal_run, archive_source, qualify_archive, MANIFEST, METADATA, PROVENANCE
|
|
|
|
|
|
class ReleaseAuditTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory(prefix='erri-release-test-')
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name)
|
|
self.write('addons/core/addon.gproj', 'Project {}')
|
|
self.write('addons/core/Scripts/Game/Test.c', 'class Test {}')
|
|
self.write('addons/core/Assets/test.png', b'nonempty resource bytes')
|
|
self.write('docs/evidence/current/manual.txt', 'Synthetic test fixture, not real acceptance.\n')
|
|
self.log = 'docs/evidence/current/console.log.txt'
|
|
self.report = 'docs/evidence/current/report.json'
|
|
self.manifest = 'docs/evidence/current/source-sha256.json'
|
|
self.write(self.log, 'ERR SMOKE PASS: synthetic\nERR SMOKE COMPLETE: synthetic\n')
|
|
self.refresh_native()
|
|
ordinary = {'status': 'passed', 'evidence': ['docs/evidence/current/manual.txt']}
|
|
native = {'status': 'passed', 'evidence': [self.report, self.manifest],
|
|
'source_manifest': self.manifest, 'native_report': self.report}
|
|
self.ledger = {'target': 'Everon, vanilla factions, 12 players',
|
|
'cases': {name: copy.deepcopy(ordinary) for name in gate.REQUIRED_CASES},
|
|
'development_checks': {'clean_native_runtime': copy.deepcopy(native)}}
|
|
self.ledger['cases']['independent_native_build'] = copy.deepcopy(native)
|
|
|
|
def write(self, relative, value):
|
|
path = self.root / relative
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(value if isinstance(value, bytes) else value.encode())
|
|
|
|
def json(self, relative, value):
|
|
self.write(relative, json.dumps(value))
|
|
|
|
def refresh_native(self):
|
|
captured = snapshot(self.root, ['synthetic-runner-not-an-engine'])
|
|
archive = Path(self.manifest).parent
|
|
self.write(self.manifest, captured[0])
|
|
self.write(str(archive / MANIFEST), captured[0])
|
|
self.write(str(archive / METADATA), captured[1])
|
|
self.json(str(archive / PROVENANCE), {'schema_version': 1, 'kind': 'launch_snapshot'})
|
|
self.json(str(archive / 'runner.json.txt'), {**bindings(captured), 'started_unix': json.loads(captured[1])['captured_unix'] + 1, 'result': 'smoke-completed', 'rounds_required': 1, 'stopped_unix': json.loads(captured[1])['captured_unix'] + 2, 'source_unchanged_after_run': True, 'end_source_sha256': hashlib.sha256(captured[0]).hexdigest(), 'console_sha256': hashlib.sha256((self.root / self.log).read_bytes()).hexdigest()})
|
|
self.json(self.report, analyze((self.root / self.log).read_text(), self.log))
|
|
|
|
def run_audit(self):
|
|
self.json('docs/acceptance.json', self.ledger)
|
|
return gate.audit(self.root)
|
|
|
|
def blocked(self, fragment):
|
|
errors = self.run_audit()
|
|
self.assertTrue(errors, 'malformed evidence incorrectly qualified')
|
|
self.assertTrue(any(fragment in error for error in errors), errors)
|
|
|
|
def test_complete_synthetic_fixture_qualifies(self):
|
|
self.assertEqual(self.run_audit(), [])
|
|
|
|
def test_failed_clean_runtime_blocks_even_when_all_cases_pass(self):
|
|
self.ledger['development_checks']['clean_native_runtime']['status'] = 'failed'
|
|
self.blocked('clean_native_runtime: status is failed')
|
|
|
|
def test_every_required_case_is_mandatory(self):
|
|
for name in gate.REQUIRED_CASES:
|
|
with self.subTest(name=name):
|
|
saved = self.ledger['cases'].pop(name)
|
|
self.blocked(name + ': required first-release case is missing')
|
|
self.ledger['cases'][name] = saved
|
|
|
|
def test_unknown_case_rejected(self):
|
|
self.ledger['cases']['fake_replacement'] = {'status': 'passed', 'evidence': [self.log]}
|
|
self.blocked('unknown first-release case')
|
|
|
|
def test_missing_runtime_gate_rejected(self):
|
|
self.ledger['development_checks'].clear()
|
|
self.blocked('required clean-runtime gate is missing')
|
|
|
|
def test_unknown_and_malformed_statuses_rejected_everywhere(self):
|
|
entry = self.ledger['cases']['escape_loop']
|
|
for status in ('PASS', 'waived', '', None, True, {}, []):
|
|
with self.subTest(status=status):
|
|
entry['status'] = status
|
|
self.blocked('unknown or malformed status')
|
|
entry['status'] = 'passed'
|
|
self.ledger['development_checks']['historical'] = {'status': 'waived', 'evidence': []}
|
|
self.blocked('unknown or malformed status')
|
|
|
|
def test_malformed_sections_entries_and_evidence(self):
|
|
original = copy.deepcopy(self.ledger)
|
|
for section in ('cases', 'development_checks'):
|
|
self.ledger = copy.deepcopy(original)
|
|
self.ledger[section] = []
|
|
self.blocked(section + ' must be an object')
|
|
for entry in (None, [], 'passed'):
|
|
self.ledger = copy.deepcopy(original)
|
|
self.ledger['cases']['escape_loop'] = entry
|
|
self.blocked('entry must be an object')
|
|
for evidence in (None, self.log, {}, [None], [42]):
|
|
self.ledger = copy.deepcopy(original)
|
|
self.ledger['cases']['escape_loop']['evidence'] = evidence
|
|
self.blocked('evidence')
|
|
|
|
def test_passed_requires_nonempty_evidence(self):
|
|
self.ledger['cases']['escape_loop']['evidence'] = []
|
|
self.blocked('requires nonempty evidence')
|
|
|
|
def test_empty_files_and_whitespace_cannot_qualify(self):
|
|
for value in (b'', b' \r\n\t'):
|
|
self.write('docs/evidence/empty.txt', value)
|
|
self.ledger['cases']['escape_loop']['evidence'] = ['docs/evidence/empty.txt']
|
|
self.blocked('evidence is empty')
|
|
|
|
def test_missing_and_directory_evidence_rejected(self):
|
|
for path in ('docs/evidence/missing.txt', 'docs/evidence/current'):
|
|
self.ledger['cases']['escape_loop']['evidence'] = [path]
|
|
self.blocked('not an existing regular file')
|
|
|
|
def test_unsafe_paths_rejected(self):
|
|
for path in ('/etc/passwd', '../outside', 'docs/../docs/evidence/current/manual.txt',
|
|
'./docs/evidence/current/manual.txt', 'docs//evidence/current/manual.txt',
|
|
'C:\\evidence.txt', '.git/config', 'docs/evidence/\x00invalid'):
|
|
with self.subTest(path=path):
|
|
self.ledger['cases']['escape_loop']['evidence'] = [path]
|
|
self.assertTrue(self.run_audit())
|
|
|
|
def test_symlinks_even_inside_repository_rejected(self):
|
|
for target in (self.root / self.log, Path('/etc/passwd')):
|
|
link = self.root / 'docs/evidence/link.txt'
|
|
link.symlink_to(target)
|
|
self.ledger['cases']['escape_loop']['evidence'] = ['docs/evidence/link.txt']
|
|
self.assertTrue(self.run_audit())
|
|
link.unlink()
|
|
|
|
def test_symlink_loop_returns_error_without_crash(self):
|
|
link = self.root / 'docs/evidence/loop'
|
|
link.symlink_to(link)
|
|
self.ledger['cases']['escape_loop']['evidence'] = ['docs/evidence/loop']
|
|
self.assertTrue(self.run_audit())
|
|
|
|
def test_duplicate_paths_rejected(self):
|
|
self.ledger['cases']['escape_loop']['evidence'] = [self.log, self.log]
|
|
self.blocked('duplicate evidence path')
|
|
|
|
def test_missing_explicit_native_fields_rejected(self):
|
|
for field in ('source_manifest', 'native_report'):
|
|
value = self.ledger['cases']['independent_native_build'].pop(field)
|
|
self.blocked('independent_native_build.' + field)
|
|
self.ledger['cases']['independent_native_build'][field] = value
|
|
|
|
def test_explicit_native_fields_must_be_listed_as_evidence(self):
|
|
self.ledger['cases']['independent_native_build']['evidence'] = [self.log]
|
|
self.blocked('must also appear')
|
|
|
|
def test_current_native_qualification_rejects_changed_added_deleted_files(self):
|
|
path = self.root / 'addons/core/Scripts/Game/Test.c'
|
|
original = path.read_bytes()
|
|
path.write_text('class Changed {}')
|
|
self.blocked('changed=1')
|
|
path.write_bytes(original)
|
|
added = self.root / 'addons/core/Assets/new.custom-resource'
|
|
added.write_bytes(b'resource')
|
|
self.blocked('missing=1')
|
|
added.unlink()
|
|
path.unlink()
|
|
self.blocked('extra=1')
|
|
|
|
def test_clean_runtime_also_requires_current_manifest(self):
|
|
self.ledger['cases']['independent_native_build']['status'] = 'pending'
|
|
self.write('addons/core/Scripts/Game/Test.c', 'class Changed {}')
|
|
self.blocked('clean_native_runtime.source_manifest: manifest does not match')
|
|
|
|
def test_incomplete_and_malformed_manifest_rejected(self):
|
|
for manifest in ({}, [], {'addons/core/addon.gproj': 'not-sha256'}):
|
|
self.json(self.manifest, manifest)
|
|
self.blocked('manifest must be a nonempty')
|
|
self.json(self.manifest, {'addons/core/addon.gproj': '0' * 64})
|
|
self.blocked('manifest does not match')
|
|
|
|
def test_native_report_cannot_hide_raw_errors(self):
|
|
self.write(self.log, 'ERR SMOKE PASS: synthetic\nWORLD (E): retained error\nERR SMOKE COMPLETE: synthetic\n')
|
|
self.blocked('does not match analysis')
|
|
self.refresh_native()
|
|
self.blocked('no script, engine or end-screen errors')
|
|
|
|
def test_each_native_failure_category_blocks(self):
|
|
for failure in ('ERR SMOKE FAIL: assertion', 'Virtual Machine Exception',
|
|
'SCRIPT (E): exception', 'ANIMATION (E): unknown resource', 'ENGINE (F): Crashed', 'Could not find expected endscreen',
|
|
"entity won't take damage and wont replicate"):
|
|
self.write(self.log, 'ERR SMOKE PASS: test\n' + failure + '\nERR SMOKE COMPLETE: test\n')
|
|
self.refresh_native()
|
|
self.blocked('native raw log must contain')
|
|
|
|
def test_truncated_and_assertion_free_logs_rejected(self):
|
|
for text in ('ERR SMOKE PASS: test\n', 'ERR SMOKE COMPLETE: test\n', 'unrelated log\n'):
|
|
self.write(self.log, text)
|
|
self.refresh_native()
|
|
self.blocked('successful completion/assertions')
|
|
|
|
def test_native_report_types_and_missing_fields_rejected(self):
|
|
report = analyze((self.root / self.log).read_text(), self.log)
|
|
for bad in ({}, [], dict(report, smoke_completions=True), dict(report, engine_errors='')):
|
|
self.json(self.report, bad)
|
|
self.blocked('native_report')
|
|
|
|
def test_native_artifacts_cannot_be_mixed_between_archives(self):
|
|
self.write('docs/evidence/other/source-sha256.json', (self.root / self.manifest).read_bytes())
|
|
entry = self.ledger['cases']['independent_native_build']
|
|
entry['source_manifest'] = 'docs/evidence/other/source-sha256.json'
|
|
entry['evidence'].append(entry['source_manifest'])
|
|
self.blocked('same evidence archive')
|
|
|
|
def test_historical_development_manifests_need_not_match_current(self):
|
|
self.json('docs/evidence/historical/source-sha256.json', {'old/path.c': '0' * 64})
|
|
self.ledger['development_checks']['historical'] = {
|
|
'status': 'passed', 'evidence': ['docs/evidence/historical/source-sha256.json']}
|
|
self.ledger['development_checks']['expected_fault'] = {'status': 'failed', 'evidence': []}
|
|
self.assertEqual(self.run_audit(), [])
|
|
|
|
def test_generated_state_excluded_but_new_resource_formats_included(self):
|
|
self.write('addons/core/resourceDatabase.rdb', 'generated')
|
|
self.write('addons/core/resources.pak', 'generated')
|
|
self.write('addons/core/profiles/runtime.json', 'generated')
|
|
self.assertEqual(self.run_audit(), [])
|
|
self.write('addons/core/Scripts/Game/resourceDatabase.rdb_fake.c', 'class New {}')
|
|
self.blocked('missing=1')
|
|
|
|
def test_source_symlink_cannot_hide_from_manifest(self):
|
|
(self.root / 'addons/core/Scripts/Game/Alias.c').symlink_to(self.root / 'addons/core/Scripts/Game/Test.c')
|
|
self.blocked('source/resource symlink is not supported')
|
|
|
|
def test_duplicate_json_keys_and_invalid_json_fail_closed(self):
|
|
for raw in ('{"cases":{},"cases":{}}', '{', '[]', '{"target":NaN}'):
|
|
self.write('docs/acceptance.json', raw)
|
|
self.assertTrue(gate.audit(self.root))
|
|
|
|
def test_symlinked_ledger_rejected(self):
|
|
self.json('docs/real-ledger.json', self.ledger)
|
|
(self.root / 'docs/acceptance.json').symlink_to(self.root / 'docs/real-ledger.json')
|
|
self.assertTrue(gate.audit(self.root))
|
|
|
|
def test_checker_is_read_only(self):
|
|
self.json('docs/acceptance.json', self.ledger)
|
|
before = {str(path.relative_to(self.root)): path.read_bytes()
|
|
for path in self.root.rglob('*') if path.is_file()}
|
|
self.assertEqual(gate.audit(self.root), [])
|
|
after = {str(path.relative_to(self.root)): path.read_bytes()
|
|
for path in self.root.rglob('*') if path.is_file()}
|
|
self.assertEqual(after, before)
|
|
|
|
def test_archive_command_uses_shared_manifest_and_report(self):
|
|
tools = self.root / 'tools'
|
|
tools.mkdir()
|
|
for name in ('archive_native.py', 'native_report.py', 'source_manifest.py', 'native_provenance.py'):
|
|
shutil.copyfile(ROOT / 'tools' / name, tools / name)
|
|
session = self.root / 'synthetic-session'
|
|
session.mkdir()
|
|
(session / 'console.log').write_text((self.root / self.log).read_text())
|
|
archive = self.root / Path(self.manifest).parent
|
|
for name in (MANIFEST, METADATA):
|
|
shutil.copyfile(archive / name, session / name)
|
|
shutil.copyfile(archive / 'runner.json.txt', session / 'runner.json')
|
|
subprocess.run([sys.executable, str(tools / 'archive_native.py'), str(session), 'new-archive'],
|
|
capture_output=True, text=True, check=True)
|
|
archive = self.root / 'docs/evidence/new-archive'
|
|
self.assertEqual(json.loads((archive / 'source-sha256.json').read_text()), addon_manifest(self.root))
|
|
report = json.loads((archive / 'report.json').read_text())
|
|
self.assertEqual(report, analyze((self.root / report['log']).read_text(), report['log']))
|
|
|
|
def test_native_report_command_compatibility(self):
|
|
result = subprocess.run([sys.executable, str(ROOT / 'tools/native_report.py'), str(self.root / self.log)],
|
|
capture_output=True, text=True, check=True)
|
|
self.assertEqual(json.loads(result.stdout), analyze((self.root / self.log).read_text(), self.root / self.log))
|
|
|
|
def test_archive_time_provenance_cannot_qualify_native_gates(self):
|
|
self.json('docs/evidence/current/' + PROVENANCE, {'schema_version': 1, 'kind': 'archive_time_only'})
|
|
self.blocked('requires launch_snapshot provenance')
|
|
|
|
def test_missing_launch_snapshot_is_not_current_evidence(self):
|
|
(self.root / 'docs/evidence/current' / METADATA).unlink()
|
|
self.blocked('launch_provenance')
|
|
|
|
def test_modified_launch_bytes_and_runner_binding_rejected(self):
|
|
path = self.root / 'docs/evidence/current' / MANIFEST
|
|
path.write_bytes(path.read_bytes() + b'\n')
|
|
self.blocked('launch manifest byte hash')
|
|
self.refresh_native()
|
|
runner = self.root / 'docs/evidence/current/runner.json.txt'
|
|
data = json.loads(runner.read_text())
|
|
data['launch_id'] = '00000000-0000-0000-0000-000000000000'
|
|
runner.write_text(json.dumps(data))
|
|
self.blocked('runner launch binding')
|
|
|
|
def test_snapshot_after_process_start_rejected(self):
|
|
runner = self.root / 'docs/evidence/current/runner.json.txt'
|
|
data = json.loads(runner.read_text())
|
|
data['started_unix'] = 1
|
|
runner.write_text(json.dumps(data))
|
|
self.blocked('snapshot must precede process launch')
|
|
|
|
def test_runner_failure_and_incomplete_repeated_rounds_cannot_qualify(self):
|
|
runner = self.root / 'docs/evidence/current/runner.json.txt'
|
|
data = json.loads(runner.read_text())
|
|
data['result'] = 'process-exited'
|
|
runner.write_text(json.dumps(data))
|
|
self.blocked('completed runner session')
|
|
data['result'] = 'smoke-completed'
|
|
data['rounds_required'] = 2
|
|
runner.write_text(json.dumps(data))
|
|
self.blocked('successful completion/assertions')
|
|
|
|
def test_prelaunch_snapshot_remains_old_when_persisted_after_edit(self):
|
|
captured = snapshot(self.root, ['synthetic'])
|
|
self.write('addons/core/Scripts/Game/Test.c', 'class ChangedAfterLaunch {}')
|
|
session = self.root / 'synthetic-late-session'
|
|
session.mkdir()
|
|
persist(session, captured)
|
|
self.assertEqual((session / MANIFEST).read_bytes(), captured[0])
|
|
(session / 'console.log').write_text('Synthetic old run')
|
|
self.json('synthetic-late-session/runner.json', {**bindings(captured), 'started_unix': json.loads(captured[1])['captured_unix'] + 1, 'stopped_unix': json.loads(captured[1])['captured_unix'] + 2, 'source_unchanged_after_run': True, 'end_source_sha256': hashlib.sha256(captured[0]).hexdigest(), 'console_sha256': hashlib.sha256((session / 'console.log').read_bytes()).hexdigest()})
|
|
with self.assertRaisesRegex(ValueError, 'differ from pre-launch snapshot'):
|
|
archive_source(session, self.root)
|
|
|
|
def test_legacy_archive_requires_opt_in_and_is_marked_historical(self):
|
|
session = self.root / 'synthetic-legacy'
|
|
session.mkdir()
|
|
with self.assertRaisesRegex(ValueError, 'no pre-launch source snapshot'):
|
|
archive_source(session, self.root)
|
|
manifest, provenance, launched = archive_source(session, self.root, allow_legacy=True)
|
|
self.assertFalse(launched)
|
|
self.assertEqual(provenance['kind'], 'archive_time_only')
|
|
self.assertEqual(provenance['qualification'], 'historical_only')
|
|
self.assertEqual(json.loads(manifest), addon_manifest(self.root))
|
|
|
|
def test_partial_launch_evidence_cannot_fall_back_to_legacy(self):
|
|
session = self.root / 'synthetic-partial'
|
|
session.mkdir()
|
|
(session / METADATA).write_text('{}')
|
|
with self.assertRaisesRegex(ValueError, 'missing or symlinked launch evidence'):
|
|
archive_source(session, self.root, allow_legacy=True)
|
|
|
|
def test_source_manifest_copy_must_match_exact_launch_bytes(self):
|
|
path = self.root / self.manifest
|
|
path.write_bytes(path.read_bytes() + b'\n')
|
|
self.blocked('differs from exact launch snapshot bytes')
|
|
|
|
def test_recomputed_report_cannot_relabel_a_different_old_raw_log(self):
|
|
self.write(self.log, 'ERR SMOKE PASS: different synthetic run\nERR SMOKE COMPLETE: other\n')
|
|
self.json(self.report, analyze((self.root / self.log).read_text(), self.log))
|
|
self.blocked('raw native log differs from stopped runner log hash')
|
|
|
|
def test_report_cannot_use_an_unsealed_neighbor_log(self):
|
|
other = 'docs/evidence/current/other.log.txt'
|
|
self.write(other, 'ERR SMOKE PASS: unrelated\nERR SMOKE COMPLETE: unrelated\n')
|
|
self.json(self.report, analyze((self.root / other).read_text(), other))
|
|
self.blocked('exact console.log.txt sealed by the stopped runner')
|
|
|
|
def test_stopped_source_change_blocks_even_if_current_files_match_launch(self):
|
|
runner = self.root / 'docs/evidence/current/runner.json.txt'
|
|
data = json.loads(runner.read_text())
|
|
data['source_unchanged_after_run'] = False
|
|
data['end_source_sha256'] = '0' * 64
|
|
runner.write_text(json.dumps(data))
|
|
self.blocked('changed between launch and stopped-run snapshot')
|
|
|
|
def test_seal_uses_stopped_sources_and_exact_log_bytes(self):
|
|
session = self.root / 'synthetic-seal'
|
|
session.mkdir()
|
|
captured = snapshot(self.root, ['synthetic'])
|
|
(session / 'console.log').write_bytes(b'exact synthetic log\r\n')
|
|
sealed = seal_run(self.root, session, captured)
|
|
self.assertTrue(sealed['source_unchanged_after_run'])
|
|
self.assertEqual(sealed['console_sha256'], hashlib.sha256(b'exact synthetic log\r\n').hexdigest())
|
|
self.write('addons/core/Scripts/Game/Test.c', 'class ChangedDuringRun {}')
|
|
self.assertFalse(seal_run(self.root, session, captured)['source_unchanged_after_run'])
|
|
|
|
def test_archive_cli_requires_explicit_legacy_import(self):
|
|
tools = self.root / 'tools'
|
|
tools.mkdir()
|
|
for name in ('archive_native.py', 'native_report.py', 'source_manifest.py', 'native_provenance.py'):
|
|
shutil.copyfile(ROOT / 'tools' / name, tools / name)
|
|
session = self.root / 'synthetic-legacy-command'
|
|
session.mkdir()
|
|
(session / 'console.log').write_text((self.root / self.log).read_text())
|
|
command = [sys.executable, str(tools / 'archive_native.py'), str(session), 'legacy-archive']
|
|
rejected = subprocess.run(command, capture_output=True, text=True)
|
|
self.assertEqual(rejected.returncode, 2)
|
|
self.assertFalse((self.root / 'docs/evidence/legacy-archive').exists())
|
|
subprocess.run(command + ['--allow-legacy-archive-time-source'], capture_output=True, text=True, check=True)
|
|
provenance = json.loads((self.root / 'docs/evidence/legacy-archive' / PROVENANCE).read_text())
|
|
self.assertEqual(provenance['kind'], 'archive_time_only')
|
|
with self.assertRaisesRegex(ValueError, 'requires launch_snapshot provenance'):
|
|
qualify_archive(self.root / 'docs/evidence/legacy-archive', addon_manifest(self.root))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|