jakub
6a667d8deb
Include the current addon, terrain integrations, tools, regression tests and recorded development evidence.
232 lines
11 KiB
Python
232 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Archive stopped TTS benchmark runs, verifying their original launch bytes.
|
|
|
|
Usage: python3 docs/evidence/tts-lag-2026-10-13/archive.py a01 a02
|
|
Append a later stopped run by supplying its tag. Existing run files are immutable.
|
|
This script neither launches an engine nor contacts a speech service.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
from collections import Counter
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
import wave
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
ROOT = HERE.parents[2]
|
|
sys.path.insert(0, str(ROOT / "tools"))
|
|
from native_provenance import archive_source, encoded, read_launch
|
|
from native_report import analyze
|
|
|
|
RELEVANT = [
|
|
"addons/core/Scripts/Game/ERR/00_Core/ERR_WelcomeVoice.c",
|
|
"addons/core/Scripts/Game/ERR/00_Core/ERR_SpeechService.c",
|
|
"addons/core/Scripts/Game/ERR/00_Core/ERR_SpeechMenu.c",
|
|
"addons/core/Scripts/Game/ERR/00_Core/ERR_AdminMenu.c",
|
|
"addons/core/Scripts/Game/ERR/00_Core/ERR_Settings.c",
|
|
"addons/core/Scripts/Game/ERR/10_AI/ERR_SoldierSpeech.c",
|
|
"addons/core/Scripts/Game/ERR/00_Core/ERR_ZSpeechBenchmark.c",
|
|
"addons/core/Configs/ERR/Tests/LegacySites.conf",
|
|
]
|
|
CLIENT_EVIDENCE = (
|
|
"client-events.txt", "log-findings.json", "admin-warning-fix.patch",
|
|
"admin-warning-validation.txt",
|
|
)
|
|
|
|
|
|
def sha256(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def write_new(path, data):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("xb") as stream:
|
|
stream.write(data)
|
|
assert path.read_bytes() == data
|
|
|
|
|
|
def copy(source, destination):
|
|
assert source.is_file() and not source.is_symlink(), source
|
|
write_new(destination, source.read_bytes())
|
|
|
|
|
|
def write_or_verify(path, data):
|
|
if path.exists():
|
|
assert path.read_bytes() == data, f"Existing evidence differs: {path}"
|
|
else:
|
|
write_new(path, data)
|
|
|
|
|
|
def events(text):
|
|
records = []
|
|
for number, line in enumerate(text.splitlines(), 1):
|
|
match = re.search(r"ERR TTS ([A-Z ]+): (.*)", line)
|
|
if not match:
|
|
continue
|
|
values = {}
|
|
for key, value in re.findall(r"(\w+)=([^ ]+)", match[2]):
|
|
try:
|
|
values[key] = int(value)
|
|
except ValueError:
|
|
try:
|
|
values[key] = float(value)
|
|
except ValueError:
|
|
values[key] = value
|
|
records.append({"console_line": number, "timestamp": line[:12],
|
|
"kind": match[1], "values": values, "raw": line})
|
|
return records
|
|
|
|
|
|
def archive(tag):
|
|
profile = Path("/tmp/erri-tts-bench-run-" + tag)
|
|
source = Path("/tmp/erri-tts-bench-source-" + tag)
|
|
sessions = list((profile / "logs").glob("logs_*"))
|
|
assert len(sessions) == 1, (profile, sessions)
|
|
session = sessions[0]
|
|
manifest, metadata, runner = read_launch(session)
|
|
assert runner["cleanup_confirmed"] is True
|
|
assert runner["cleanup_report"]["idle_verified"] is True
|
|
assert runner["external_addons_unchanged_after_run"] is True
|
|
manifest_bytes, provenance, launched = archive_source(session, source)
|
|
assert launched and metadata["source_root"] == str(source)
|
|
for name in RELEVANT:
|
|
assert sha256((source / name).read_bytes()) == manifest[name], name
|
|
destination = HERE / tag
|
|
destination.mkdir(exist_ok=False)
|
|
for name in ("console.log", "script.log", "error.log", "runner.json",
|
|
"external-addons-inputs.json", "headless-launch.log"):
|
|
copy(session / name, destination / (name + ".txt"))
|
|
for name in ("launch-source-sha256.json", "launch-source.json"):
|
|
copy(session / name, destination / name)
|
|
write_new(destination / "source-sha256.json", manifest_bytes)
|
|
write_new(destination / "source-provenance.json", encoded(provenance))
|
|
source_matches = {}
|
|
for name in RELEVANT:
|
|
copy(source / name, destination / "source" / name)
|
|
current = ROOT / name
|
|
source_matches[name] = {
|
|
"launch_sha256": manifest[name],
|
|
"archived_sha256": sha256((destination / "source" / name).read_bytes()),
|
|
"repository_exists_at_archive": current.is_file(),
|
|
"repository_matches_launch_at_archive": current.is_file() and sha256(current.read_bytes()) == manifest[name],
|
|
}
|
|
write_new(destination / "relevant-source-matches.json", encoded(source_matches))
|
|
for name in ("samples.json", "production-sources.json"):
|
|
if (profile / name).exists():
|
|
copy(profile / name, destination / name)
|
|
runner_output = Path("/tmp/erri-tts-bench-" + tag + "-runner.log")
|
|
if runner_output.exists():
|
|
copy(runner_output, destination / "runner-output.log.txt")
|
|
samples = {}
|
|
for path in sorted((profile / "profile").glob("tts-*.b64")):
|
|
raw = path.read_bytes()
|
|
pcm = base64.b64decode(raw, validate=True)
|
|
seconds = int(path.stem.split("-")[1])
|
|
with wave.open(io.BytesIO(pcm), "rb") as wav:
|
|
assert wav.getnchannels() == 1 and wav.getsampwidth() == 2
|
|
assert wav.getframerate() == 24000 and wav.getnframes() == seconds * 24000
|
|
assert not any(wav.readframes(wav.getnframes())), "Fixture must contain synthetic silence only"
|
|
write_new(destination / "inputs" / (path.name + ".gz"), gzip.compress(raw, mtime=0))
|
|
assert gzip.decompress((destination / "inputs" / (path.name + ".gz")).read_bytes()) == raw
|
|
samples[str(seconds)] = {
|
|
"base64_bytes": len(raw), "base64_sha256": sha256(raw),
|
|
"wav_bytes": len(pcm), "wav_sha256": sha256(pcm),
|
|
"format": "RIFF PCM mono 24000 Hz 16 bit", "synthetic_silence": True,
|
|
}
|
|
outputs = [path.with_suffix(".wav"), path.with_name("tts-new-output-" + str(seconds) + ".wav")]
|
|
for output in outputs:
|
|
if not output.exists():
|
|
continue
|
|
assert output.read_bytes() == pcm, f"Decoded WAV differs from input: {output}"
|
|
samples[str(seconds)].setdefault("native_outputs", {})[output.name] = {
|
|
"matches_input": True, "sha256": sha256(output.read_bytes()),
|
|
}
|
|
write_new(destination / "outputs" / (output.name + ".gz"), gzip.compress(output.read_bytes(), mtime=0))
|
|
write_new(destination / "input-output-sha256.json", encoded(samples))
|
|
text = (session / "console.log").read_text(errors="replace")
|
|
report = analyze(text, str((destination / "console.log.txt").relative_to(ROOT)))
|
|
report["scope"] = "Local Workbench script timing with synthetic PCM silence; no service contact, network latency, audible quality or multiplayer qualification. 1 ms timer resolution."
|
|
write_new(destination / "native-report.json", encoded(report))
|
|
write_new(destination / "timings.json", encoded(events(text)))
|
|
hashes = {str(path.relative_to(destination)): sha256(path.read_bytes())
|
|
for path in sorted(destination.rglob("*")) if path.is_file()}
|
|
write_new(destination / "artifact-sha256.json", encoded(hashes))
|
|
print(f"Archived {tag}: {runner['result']}, {len(events(text))} timing records, cleanup verified")
|
|
|
|
|
|
def summarize():
|
|
runs = {}
|
|
manifests = {}
|
|
errors = {}
|
|
for folder in sorted(HERE.glob("a[0-9][0-9]")):
|
|
if not folder.is_dir():
|
|
continue
|
|
manifest, metadata, runner = read_launch(folder, "runner.json.txt", "console.log.txt")
|
|
report = json.loads((folder / "native-report.json").read_text())
|
|
hashes = json.loads((folder / "artifact-sha256.json").read_text())
|
|
assert all(sha256((folder / name).read_bytes()) == digest for name, digest in hashes.items())
|
|
manifests[folder.name] = manifest
|
|
errors[folder.name] = Counter(re.sub(r"\s+", " ", re.sub(r"^\d\d:\d\d:\d\d\.\d+\s+", "", line)) for line in report["engine_errors"])
|
|
runs[folder.name] = {
|
|
"runner_result": runner["result"], "cleanup_confirmed": runner["cleanup_confirmed"],
|
|
"external_addons_unchanged_after_run": runner["external_addons_unchanged_after_run"],
|
|
"assertions_passed": report["assertions_passed"],
|
|
"smoke_completions": report["smoke_completions"],
|
|
"script_or_runtime_failure_count": len(report["script_or_runtime_failures"]),
|
|
"engine_error_count": len(report["engine_errors"]),
|
|
"timings": json.loads((folder / "timings.json").read_text()),
|
|
}
|
|
starts = [event for event in runs[folder.name]["timings"] if event["kind"] == "NEW START"]
|
|
if starts:
|
|
def milliseconds(timestamp):
|
|
hours, minutes, seconds = timestamp.split(":")
|
|
return round((int(hours) * 3600 + int(minutes) * 60 + float(seconds)) * 1000)
|
|
started = milliseconds(starts[0]["timestamp"])
|
|
runs[folder.name]["queued_batch_playback_delay_ms"] = {
|
|
str(event["values"]["seconds"]): (milliseconds(event["timestamp"]) - started) % 86400000
|
|
for event in runs[folder.name]["timings"] if event["kind"] == "NEW PLAY"
|
|
}
|
|
changes = {}
|
|
error_changes = {}
|
|
tags = sorted(manifests)
|
|
for previous, current in zip(tags, tags[1:]):
|
|
one, two = manifests[previous], manifests[current]
|
|
changes[previous + "_to_" + current] = sorted(name for name in set(one) | set(two) if one.get(name) != two.get(name))
|
|
error_changes[previous + "_to_" + current] = {
|
|
"new_signatures": sorted(set(errors[current]) - set(errors[previous])),
|
|
"removed_signatures": sorted(set(errors[previous]) - set(errors[current])),
|
|
"additional_occurrences": dict(errors[current] - errors[previous]),
|
|
}
|
|
report = {
|
|
"scope": "Timing measured inside native Enforce script callbacks. This is component preparation cost, not a rendered-frame percentile or network benchmark.",
|
|
"runs": runs, "changed_launch_sources": changes, "engine_error_comparison": error_changes,
|
|
"client_log_evidence": "client/log-findings.json",
|
|
"client_limitations": "Download-to-play gaps include network and preparation; no per-frame client telemetry was retained, so those gaps alone do not establish render hitches.",
|
|
"audio_limitations": "Synthetic silence is terminated immediately in baseline runs; timing does not qualify audible speech, an entire utterance, or actual multiplayer streaming.",
|
|
"queued_measurement_limitations": "Queued fixtures inject local payloads before timing. Shared-batch playback waits are not single-clip or network latency. maxPrepareMs covers only one preparation call; maxScriptUpdateMs and maxFrameGapMs also include other active mission work, so the change does not prove all game frames are hitch-free.",
|
|
"provenance": "read_launch validates original capture, stopped console hash and unchanged source manifest. archive_source verifies full frozen addon bytes. Per-run artifact hashes bind preserved evidence; runner confirms cleanup and unchanged external addon inputs.",
|
|
}
|
|
(HERE / "report.json").write_bytes(encoded(report))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("runs", nargs="*", help="Stopped benchmark tags, e.g. a01 a02 a03")
|
|
args = parser.parse_args()
|
|
for tag in args.runs:
|
|
if not re.fullmatch(r"a[0-9]{2}", tag):
|
|
parser.error("Expected a run tag such as a03")
|
|
archive(tag)
|
|
for name in CLIENT_EVIDENCE:
|
|
write_or_verify(HERE / "client" / name, (ROOT / ".local/tts-lag-20261013" / name).read_bytes())
|
|
summarize()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|