425 lines
22 KiB
Python
425 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Build unqualified native PAK candidates in a fresh private /tmp session.
|
|
Never publish. Native package contents and consumption remain separate checks.
|
|
"""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
import zipfile
|
|
|
|
from source_bundle import build, inventory, PROJECTS
|
|
from headless_game import assert_no_game, assert_prefix_idle, cleanup_owned_run
|
|
from native_report import analyze
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
ORDER = tuple(PROJECTS)
|
|
# Record the direct Proton/Wine status independently of Gamescope's exit status.
|
|
# This helper lives only in the owned session and never interprets shell text.
|
|
CHILD_RECORDER = """import json, os, subprocess, sys
|
|
from pathlib import Path
|
|
command = sys.argv[3:]
|
|
environment = os.environ.copy()
|
|
environment.update(json.loads(Path(sys.argv[2]).read_text()))
|
|
code = subprocess.call(command, env=environment)
|
|
with Path(sys.argv[1]).open('x') as output:
|
|
json.dump({'command': command, 'exit_code': code}, output)
|
|
sys.exit(code)
|
|
"""
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with path.open('rb') as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b''):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def save(path, value):
|
|
path.write_text(json.dumps(value, indent=2, sort_keys=True) + '\n')
|
|
|
|
|
|
def project_order(project):
|
|
if project not in PROJECTS:
|
|
raise ValueError('Unknown successor project')
|
|
needed = {project, *PROJECTS[project][2]}
|
|
return [name for name in ORDER if name in needed]
|
|
|
|
|
|
def validate_session(root, steam, compat, session):
|
|
if session.exists() or session.is_symlink():
|
|
raise ValueError('--session must not already exist; outputs are never reused')
|
|
if session != session.resolve() or session.parent != Path('/tmp') or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,119}', session.name):
|
|
raise ValueError('--session must be a fresh direct child of /tmp')
|
|
if any(session.is_relative_to(p) or p.is_relative_to(session) for p in (root, steam, compat)):
|
|
raise ValueError('Session must be separate from source, Steam and prefix')
|
|
if (not compat.is_relative_to(Path('/tmp')) or compat.is_relative_to(root)
|
|
or compat.is_relative_to(steam) or (compat/'pfx').is_symlink()
|
|
or (compat/'pfx/system.reg').is_symlink() or not (compat/'pfx/system.reg').is_file()):
|
|
raise ValueError('--compat must be a disposable cloned prefix under /tmp, outside source and Steam, with a real pfx')
|
|
if (compat/'.erri-workbench.lock').is_symlink():
|
|
raise ValueError('Disposable prefix lock must not be a symlink')
|
|
|
|
|
|
def copy_payload(root, session):
|
|
# Same reviewed authored schema, resource identities and dependency checks as
|
|
# source_bundle; no caches, original content, logs or vanilla bytes copied.
|
|
archive, manifest = build(root)
|
|
authored = {name: sha for name, sha in manifest['files'].items() if name.startswith('addons/')}
|
|
session.mkdir(mode=0o700) # atomic exclusive ownership; never exist_ok
|
|
source = session/'source'
|
|
with zipfile.ZipFile(io.BytesIO(archive)) as bundle:
|
|
for name, sha in authored.items():
|
|
data = bundle.read(name)
|
|
if hashlib.sha256(data).hexdigest() != sha:
|
|
raise ValueError('Validated bundle hash mismatch')
|
|
destination = source/name
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
with destination.open('xb') as output:
|
|
output.write(data)
|
|
save(session/'authored-before.json', authored)
|
|
return authored
|
|
|
|
|
|
def copied_authored_snapshot(source, authored):
|
|
result = {}
|
|
for name in authored:
|
|
path = source/name
|
|
result[name] = digest(path) if path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(source.resolve()) else None
|
|
# The shared inventory rejects unreviewed additions and ignores only its
|
|
# explicit generated-cache schema. Added authored scripts/resources matter.
|
|
discovered = inventory(source)
|
|
for name, sha in discovered.items():
|
|
if name not in result:
|
|
result[name] = sha
|
|
return result
|
|
|
|
|
|
def copied_authored_unchanged(source, authored):
|
|
return copied_authored_snapshot(source, authored) == authored
|
|
|
|
|
|
def command_for(steam, session, project, silent_workbench=False, exit_after_init=False):
|
|
if silent_workbench and exit_after_init:
|
|
raise ValueError("Choose only one Workbench exit mode")
|
|
if project not in PROJECTS:
|
|
raise ValueError('Unknown successor project')
|
|
command = ['gamescope', '--backend', 'headless', '-W', '1280', '-H', '720',
|
|
'-w', '1280', '-h', '720', '-r', '30', '--',
|
|
str(steam/'steamapps/common/Proton - Experimental/proton'), 'run',
|
|
str(steam/'steamapps/common/Arma Reforger Tools/Workbench/ArmaReforgerWorkbenchSteamDiag.exe'),
|
|
'-gproj', 'Z:'+str(session/'source/addons'/project/'addon.gproj'),
|
|
'-addonsDir', 'Z:'+str(session/'source/addons'),
|
|
'-profile', 'Z:'+str(session/'profiles'/project),
|
|
'-logconsole', '-noBackend', '-noSound', '-disableCrashReporter',
|
|
'-wbModule=ResourceManager', '-packAddon',
|
|
'-packAddonDir', 'Z:'+str(session/'packages'/project),
|
|
'-packAddonCacheDir', 'Z:'+str(session/'cache'/project)]
|
|
if silent_workbench:
|
|
command.append('-wbSilent')
|
|
elif exit_after_init:
|
|
command.append('-exitAfterInit')
|
|
return command
|
|
|
|
|
|
|
|
def child_status(path, expected):
|
|
if path.is_symlink() or not path.is_file():
|
|
return None
|
|
try:
|
|
record = json.loads(path.read_text())
|
|
except (OSError, ValueError):
|
|
return None
|
|
if not isinstance(record, dict) or record.get('command') != expected or type(record.get('exit_code')) is not int:
|
|
return None
|
|
return record['exit_code']
|
|
|
|
|
|
def log_evidence(paths):
|
|
records, failures = {}, []
|
|
for path in sorted(paths):
|
|
if path.is_symlink() or not path.is_file():
|
|
raise ValueError('Unexpected nonregular native log')
|
|
content = path.read_text(errors='replace')
|
|
report = analyze(content, path)
|
|
errors = report['engine_errors'] + report['script_or_runtime_failures'] + report['missing_end_screen']
|
|
errors += [line.strip() for line in content.splitlines()
|
|
if re.search(r"can't compile|packing (?:has )?failed|packAddon.*failed|fatal error", line, re.I)]
|
|
if path.name in {'error.log', 'crash.log'} and content.strip():
|
|
errors.append('Nonempty native '+path.name)
|
|
records[str(path)] = {'sha256': digest(path), 'bytes': path.stat().st_size, 'errors': errors}
|
|
failures.extend(errors)
|
|
return records, failures
|
|
|
|
|
|
def package_inventory(folder):
|
|
records = {}
|
|
if folder.is_symlink():
|
|
raise ValueError('Native package root is a symlink')
|
|
if not folder.is_dir():
|
|
return records
|
|
for path in folder.rglob('*'):
|
|
if path.is_symlink() or not (path.is_file() or path.is_dir()):
|
|
raise ValueError('Nonregular native package output')
|
|
if path.is_file():
|
|
records[path.relative_to(folder).as_posix()] = {'bytes': path.stat().st_size, 'sha256': digest(path)}
|
|
return records
|
|
|
|
|
|
def candidate_ok(exit_code, result, cleanup_ok, unchanged, logs, errors, packages):
|
|
return (exit_code == 0 and result in {'process-exited', 'native-child-exited'} and cleanup_ok and unchanged
|
|
and bool(logs) and not errors
|
|
and any(name.lower().endswith('.pak') and info['bytes'] > 0 for name, info in packages.items())
|
|
and all(info['bytes'] > 0 for name, info in packages.items() if name.lower().endswith('.pak')))
|
|
|
|
|
|
def quiet_streams(application_id, sink, compat):
|
|
streams = json.loads(subprocess.check_output(['pactl', '-f', 'json', 'list', 'sink-inputs'], text=True, timeout=5))
|
|
for stream in streams:
|
|
props = stream.get('properties', {})
|
|
owned = props.get('application.id') == application_id
|
|
pid = str(props.get('application.process.id', ''))
|
|
if not owned and pid.isdigit():
|
|
try:
|
|
owned = ('WINEPREFIX='+str(compat/'pfx')).encode() in (Path('/proc')/pid/'environ').read_bytes().split(b'\0')
|
|
except (FileNotFoundError, PermissionError, ProcessLookupError):
|
|
pass
|
|
if owned:
|
|
for args in [['move-sink-input', str(stream['index']), sink], ['set-sink-input-mute', str(stream['index']), '1']]:
|
|
subprocess.run(['pactl', *args], check=True, timeout=5, stdout=subprocess.DEVNULL)
|
|
|
|
|
|
def run_project(steam, compat, session, project, timeout, authored, silent_workbench=False, exit_after_init=False):
|
|
command = command_for(steam, session, project, silent_workbench, exit_after_init)
|
|
proton = Path(command[command.index('--')+1])
|
|
executable = Path(command[command.index('--')+3])
|
|
profile = session/'profiles'/project
|
|
(profile/'addons').mkdir(parents=True)
|
|
(profile/'addons/ArmaReforger').symlink_to(steam/'steamapps/common/Arma Reforger/addons/data', target_is_directory=True)
|
|
for name in ('packages', 'cache', 'evidence'):
|
|
(session/name/project).mkdir(parents=True)
|
|
evidence = session/'evidence'/project
|
|
separator = command.index('--')
|
|
native_command = command[separator+1:]
|
|
recorder = evidence/'native-child.py'
|
|
recorder.write_text(CHILD_RECORDER)
|
|
status_path = evidence/'native-exit.json'
|
|
environment_path = evidence/'native-prefix-environment.json'
|
|
prefix_environment = {'STEAM_COMPAT_DATA_PATH': str(compat), 'STEAM_COMPAT_CLIENT_INSTALL_PATH': str(steam),
|
|
'WINEPREFIX': str(compat/'pfx')}
|
|
save(environment_path, prefix_environment)
|
|
command = command[:separator+1] + [sys.executable, str(recorder), str(status_path), str(environment_path), *native_command]
|
|
save(evidence/'launch.json', {'command': command, 'native_command': native_command, 'authored': authored, 'started_unix': time.time(),
|
|
'executable_sha256': digest(executable), 'proton_sha256': digest(proton),
|
|
'child_recorder_sha256': digest(recorder), 'native_environment_sha256': digest(environment_path),
|
|
'silent_workbench_experiment': silent_workbench,
|
|
'exit_after_init_experiment': exit_after_init})
|
|
run_id = uuid.uuid4().hex
|
|
sink, application_id = 'erri_pack_'+run_id, 'org.escapistsrereintroduced.pack.'+run_id
|
|
env = os.environ.copy()
|
|
env.update(prefix_environment)
|
|
env.update(PULSE_SINK=sink, PULSE_PROP='application.id='+application_id)
|
|
# Prefix-idle checks must not mistake a lingering display wrapper for Wine.
|
|
# Only the recorder's direct Proton child receives these three overrides.
|
|
display_env = {key: value for key, value in env.items() if key not in prefix_environment}
|
|
child = None
|
|
module = None
|
|
exit_code = None
|
|
result, cleanup_ok, runner_error = 'timeout', True, None
|
|
cleanup_report = None
|
|
launch_log = evidence/'launch.log'
|
|
deadline = time.monotonic()+timeout
|
|
launcher_exited_at = None
|
|
try:
|
|
assert_no_game()
|
|
assert_prefix_idle(compat)
|
|
module = subprocess.check_output(['pactl', 'load-module', 'module-null-sink', 'sink_name='+sink], text=True, timeout=5).strip()
|
|
for args in [['set-sink-mute', sink, '1'], ['set-sink-volume', sink, '0%']]:
|
|
subprocess.run(['pactl', *args], check=True, timeout=5)
|
|
assert_prefix_idle(compat)
|
|
with launch_log.open('x') as output:
|
|
child = subprocess.Popen(command, cwd=executable.parent, env=display_env, stdout=output,
|
|
stderr=subprocess.STDOUT, start_new_session=True)
|
|
while time.monotonic() < deadline:
|
|
quiet_streams(application_id, sink, compat)
|
|
if time.monotonic() >= deadline:
|
|
break # audio/process observation must not extend the deadline
|
|
exit_code = child.poll()
|
|
recorded_native_exit = child_status(status_path, native_command)
|
|
if exit_code is not None or recorded_native_exit is not None:
|
|
# Gamescope exit alone cannot prove Workbench ended.
|
|
# Never qualify output completed by forcibly killing a child.
|
|
if launcher_exited_at is None:
|
|
launcher_exited_at = time.monotonic()
|
|
try:
|
|
assert_prefix_idle(compat)
|
|
if time.monotonic() >= deadline:
|
|
break
|
|
result = 'process-exited' if exit_code is not None else 'native-child-exited'
|
|
break
|
|
except ValueError:
|
|
if time.monotonic()-launcher_exited_at >= 10:
|
|
result = 'native-or-launcher-exited-prefix-active'
|
|
break
|
|
time.sleep(min(.5, max(0, deadline-time.monotonic())))
|
|
except KeyboardInterrupt:
|
|
result = 'interrupted'
|
|
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
|
result, runner_error = 'runner-failed', str(error)
|
|
finally:
|
|
if child is not None:
|
|
try:
|
|
# Shared post-run adapter: normal shutdown first, then only its
|
|
# proven canonical adopted helpers, followed by strict idle proof.
|
|
# This never upgrades the completion result observed before cleanup.
|
|
cleanup_report = cleanup_owned_run(child, proton, env, compat)
|
|
cleanup_ok = isinstance(cleanup_report, dict) and cleanup_report.get('cleanup_confirmed') is True
|
|
if not cleanup_ok:
|
|
detail = 'Owned packaging cleanup unconfirmed'
|
|
if isinstance(cleanup_report, dict):
|
|
detail += ': ' + '; '.join(str(item) for item in cleanup_report.get('errors', []))
|
|
runner_error = (runner_error + '; ' if runner_error else '') + detail
|
|
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
|
cleanup_ok = False
|
|
runner_error = (runner_error + '; ' if runner_error else '') + 'Owned packaging cleanup: ' + str(error)
|
|
if module is not None:
|
|
try:
|
|
subprocess.run(['pactl', 'unload-module', module], check=True, timeout=5, stdout=subprocess.DEVNULL)
|
|
except (OSError, subprocess.SubprocessError) as error:
|
|
cleanup_ok, runner_error = False, str(error)
|
|
paths = list((profile/'logs').rglob('*.log'))
|
|
if launch_log.is_file():
|
|
paths.append(launch_log)
|
|
try:
|
|
if any(not path.resolve().is_relative_to(session.resolve()) for path in paths):
|
|
raise ValueError('Native log escaped owned session')
|
|
logs, errors = log_evidence(paths)
|
|
packages = package_inventory(session/'packages'/project)
|
|
after = copied_authored_snapshot(session/'source', authored)
|
|
save(evidence/'authored-after.json', after)
|
|
unchanged = after == authored
|
|
except (OSError, ValueError) as error:
|
|
logs, packages, unchanged = {}, {}, False
|
|
errors = ['Output/evidence validation failed: '+str(error)]
|
|
native_logs = [name for name in logs if Path(name).name == 'console.log' and logs[name]['bytes'] > 0]
|
|
native_exit = child_status(status_path, native_command)
|
|
wrapper_ok = exit_code == 0 or (exit_code is None and result == 'native-child-exited')
|
|
candidate = wrapper_ok and candidate_ok(native_exit, result, cleanup_ok, unchanged, native_logs, errors, packages)
|
|
report = {'project': project, 'result': 'native-pack-candidate' if candidate else ('validation-failed' if result in {'process-exited', 'native-child-exited'} else result),
|
|
'candidate_created': candidate, 'native_exit_code': native_exit, 'launcher_exit_code': exit_code,
|
|
'completion_observed': result, 'display_cleanup_after_native_exit': result == 'native-child-exited',
|
|
'silent_workbench_experiment': silent_workbench,
|
|
'exit_after_init_experiment': exit_after_init,
|
|
'native_status_sha256': digest(status_path) if status_path.is_file() and not status_path.is_symlink() else None,
|
|
'cleanup_confirmed': cleanup_ok, 'cleanup_report': cleanup_report,
|
|
'copied_authored_unchanged': unchanged, 'runner_error': runner_error,
|
|
'logs': logs, 'errors': errors, 'packages': packages,
|
|
'package_contents_verified': False, 'native_consumption_verified': False,
|
|
'tool_completion_semantics_verified': False, 'release_qualified': False,
|
|
'qualification': 'PENDING: inspect native contents and load packed-only addon closure; no release claim'}
|
|
save(evidence/'report.json', report)
|
|
return report
|
|
|
|
|
|
def run(a, root=ROOT):
|
|
root, steam, compat = root.resolve(), a.steam.resolve(), a.compat.resolve()
|
|
session = a.session.absolute() # refuse existing symlinks before resolve
|
|
validate_session(root, steam, compat, session)
|
|
if not 30 <= a.timeout <= 1800:
|
|
raise ValueError('--timeout must be 30..1800 seconds per addon')
|
|
silent_workbench = getattr(a, 'silent_workbench', False)
|
|
exit_after_init = getattr(a, 'exit_after_init', False)
|
|
command = command_for(steam, session, a.project, silent_workbench, exit_after_init)
|
|
separator = command.index('--')
|
|
for path in (Path(command[separator+1]), Path(command[separator+3]), steam/'steamapps/common/Arma Reforger/addons/data/ArmaReforger.gproj'):
|
|
if not path.is_file():
|
|
raise ValueError('Installed runtime dependency missing: '+str(path))
|
|
if not shutil.which('gamescope') or not shutil.which('pactl'):
|
|
raise ValueError('gamescope and pactl required')
|
|
assert_no_game()
|
|
descriptor = os.open(compat/'.erri-workbench.lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(descriptor, 'a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise ValueError('Another native runner owns the disposable prefix')
|
|
assert_prefix_idle(compat)
|
|
authored = copy_payload(root, session)
|
|
reports = []
|
|
order = project_order(a.project)
|
|
continue_on_failure = getattr(a, 'continue_on_failure', False)
|
|
stopped_reason = None
|
|
for project in order:
|
|
report = run_project(steam, compat, session, project, a.timeout, authored, silent_workbench, exit_after_init)
|
|
reports.append(report)
|
|
# Diagnostic continuation never grants authority to reuse an unsafe
|
|
# prefix or a changed source copy. Timeout/interruption also stop.
|
|
if not report.get('cleanup_confirmed'):
|
|
stopped_reason = 'owned cleanup unconfirmed'
|
|
elif report.get('runner_error'):
|
|
stopped_reason = 'native runner error'
|
|
elif not report.get('copied_authored_unchanged'):
|
|
stopped_reason = 'copied authored source changed or unverifiable'
|
|
elif report.get('completion_observed') not in {'process-exited', 'native-child-exited'}:
|
|
stopped_reason = 'native completion not safely observed'
|
|
elif not report['candidate_created'] and not continue_on_failure:
|
|
stopped_reason = 'candidate failed; continuation not requested'
|
|
if stopped_reason:
|
|
break
|
|
if len(reports) < len(order):
|
|
try:
|
|
assert_prefix_idle(compat)
|
|
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
|
stopped_reason = 'prefix not verifiably idle before next addon: '+str(error)
|
|
break
|
|
try:
|
|
original_after = inventory(root)
|
|
save(session/'repository-after.json', original_after)
|
|
original_unchanged = original_after == authored
|
|
except (OSError, ValueError):
|
|
original_unchanged = False
|
|
summary = {'kind': 'UNQUALIFIED NATIVE PACKAGE ATTEMPT', 'session': str(session),
|
|
'project_order': order, 'projects': reports,
|
|
'continue_on_failure': continue_on_failure, 'stopped_reason': stopped_reason,
|
|
'skipped_projects': order[len(reports):],
|
|
'source_repository_unchanged': original_unchanged,
|
|
'release_qualified': False, 'published': False,
|
|
'qualification': 'PENDING native contents inspection and packed-only consumption'}
|
|
summary['all_candidates_created'] = original_unchanged and stopped_reason is None and len(reports) == len(order) and all(r['candidate_created'] for r in reports)
|
|
save(session/'package-report.json', summary)
|
|
print(json.dumps(summary, indent=2))
|
|
return 0 if summary['all_candidates_created'] else 1
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--project', choices=PROJECTS, default='core', help='Build selected addon and its successor dependencies in order')
|
|
parser.add_argument('--session', type=Path, required=True, help='New direct child of /tmp; never overwritten')
|
|
parser.add_argument('--compat', type=Path, required=True, help='Idle disposable cloned Tools prefix under /tmp')
|
|
parser.add_argument('--steam', type=Path, default=Path.home()/'.local/share/Steam')
|
|
parser.add_argument('--timeout', type=int, default=300, help='30..1800 seconds per addon')
|
|
lifecycle = parser.add_mutually_exclusive_group()
|
|
lifecycle.add_argument('--silent-workbench', action='store_true', help='Opt-in documented -wbSilent experiment; packing compatibility still requires native evidence')
|
|
lifecycle.add_argument('--exit-after-init', action='store_true', help='Opt-in documented -exitAfterInit experiment; natural exit and all native error gates remain mandatory')
|
|
parser.add_argument('--continue-on-failure', action='store_true', help='Diagnostic only: continue completed, safely cleaned addon attempts after validation failures; any failure keeps overall exit nonzero and unqualified')
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
return run(args)
|
|
except (ValueError, OSError, subprocess.SubprocessError) as error:
|
|
print(json.dumps({'result': 'refused-or-failed', 'error': str(error), 'release_qualified': False}))
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|