#!/usr/bin/env python3 """Compare declared equivalent frame/tick recordings; never grants acceptance.""" import argparse import csv import hashlib import json import math from pathlib import Path import statistics COLUMNS = ['frame_index', 'elapsed_ms', 'frame_time_ms', 'players', 'ai', 'vehicles', 'entities', 'ai_simulation_ms'] POPULATION = ['players', 'ai', 'vehicles', 'entities'] def require(condition, message): if not condition: raise ValueError(message) def keys(value, expected, label): require(isinstance(value, dict) and set(value) == set(expected), f'{label}: expected fields {expected}') def number(value, label, minimum=0): require(type(value) in (float, int) and math.isfinite(value) and value >= minimum, f'{label}: invalid number') return value def text(value, label): require(isinstance(value, str) and bool(value.strip()), f'{label}: required nonempty string') def integer(value, label, minimum=0): require(type(value) is int and value >= minimum, f'{label}: invalid integer') def finite_json(value): if isinstance(value, float): require(math.isfinite(value), 'Non-finite JSON number') elif isinstance(value, dict): for item in value.values(): finite_json(item) elif isinstance(value, list): for item in value: finite_json(item) def percentile(values, fraction): """Nearest rank, not interpolation; slow-tail frame times use upper ranks.""" return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] def metrics(frames, ai_load): fps = [1000 / value for value in frames] result = { 'median_frame_ms': statistics.median(frames), 'p95_frame_ms': percentile(frames, .95), 'p99_frame_ms': percentile(frames, .99), 'median_fps': statistics.median(fps), 'p05_fps': percentile(fps, .05), 'p01_fps': percentile(fps, .01), 'throughput_fps': 1000 * len(frames) / sum(frames), 'median_ai_simulation_ms': statistics.median(ai_load), 'p95_ai_simulation_ms': percentile(ai_load, .95), } finite_json(result) return result def load_samples(path, run): workload = run['workload'] warmup = workload['warmup_seconds'] * 1000 end = warmup + workload['duration_seconds'] * 1000 frames, ai_load = [], [] boundary_frames = [] previous = 0.0 total_frame_ms = 0.0 row_count = 0 with path.open(newline='', encoding='utf-8') as stream: reader = csv.DictReader(stream) require(reader.fieldnames == COLUMNS, f'{path}: expected exact CSV header {COLUMNS}') for index, row in enumerate(reader, 1): label = f'{path}: row {index + 1}' require(set(row) == set(COLUMNS) and all(v is not None for v in row.values()), f'{label}: incomplete/extra columns') try: frame_index = int(row['frame_index']) elapsed = float(row['elapsed_ms']) frame = float(row['frame_time_ms']) ai_ms = float(row['ai_simulation_ms']) population = {key: int(row[key]) for key in POPULATION} except (ValueError, TypeError) as error: raise ValueError(f'{label}: invalid numeric sample') from error require(frame_index == index, f'{label}: missing/duplicate/out-of-order frame index') require(all(math.isfinite(v) for v in (elapsed, frame, ai_ms)) and frame >= .001 and elapsed > previous and ai_ms >= 0, f'{label}: invalid frame/time/load') require(abs((elapsed - previous) - frame) <= .1, f'{label}: elapsed delta differs from frame time by over 0.1 ms') total_frame_ms += frame require(abs(total_frame_ms - elapsed) <= .1, f'{label}: cumulative elapsed time differs from recorded frames') require(previous < end, f'{label}: samples continue beyond the declared capture window') for key in POPULATION: require(population[key] == workload[key], f'{label}: {key} workload changed') if previous >= warmup and elapsed <= end: frames.append(frame) ai_load.append(ai_ms) else: overlap = max(0, min(elapsed, end) - max(previous, warmup)) if overlap > 0: boundary_frames.append({'frame_index': frame_index, 'frame_time_ms': frame, 'measurement_overlap_ms': overlap}) previous = elapsed row_count += 1 require(previous >= end, f'{path}: incomplete recording; warmup plus measurement duration not covered') measured_ms = sum(frames) coverage = measured_ms / (workload['duration_seconds'] * 1000) require(coverage >= .99, f'{path}: only {coverage:.3%} of declared measurement duration has complete frames; boundary stalls/intervals exclude over 1%') require(len(frames) >= 120, f'{path}: need at least 120 complete measurement frames/ticks') return {'samples_total': row_count, 'samples_measured': len(frames), 'measured_frame_seconds': measured_ms / 1000, 'measurement_coverage_fraction': coverage, 'boundary_frames_excluded': boundary_frames, 'csv_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'metrics': metrics(frames, ai_load)} def validate_run(run): keys(run, ['variant', 'repeat_id', 'artifact_id', 'addon_ids', 'engine_version', 'hardware', 'workload', 'capture', 'sample_file'], 'run') require(run['variant'] in ('baseline', 'successor'), 'run: variant must be baseline or successor') for key in ['repeat_id', 'artifact_id', 'engine_version', 'sample_file']: text(run[key], key) require(isinstance(run['addon_ids'], list) and bool(run['addon_ids']), 'addon_ids: nonempty list required') for addon in run['addon_ids']: text(addon, 'addon id') require(len(set(run['addon_ids'])) == len(run['addon_ids']), 'addon_ids: duplicates') keys(run['hardware'], ['cpu', 'gpu', 'ram_gib', 'os', 'driver'], 'hardware') for key in ['cpu', 'gpu', 'os', 'driver']: text(run['hardware'][key], f'hardware.{key}') number(run['hardware']['ram_gib'], 'hardware.ram_gib', 1) workload = run['workload'] keys(workload, ['map', 'route', *POPULATION, 'settings', 'warmup_seconds', 'duration_seconds'], 'workload') text(workload['map'], 'workload.map') text(workload['route'], 'workload.route') for key in POPULATION: integer(workload[key], f'workload.{key}', 1 if key == 'players' else 0) require(workload['entities'] >= workload['players'] + workload['ai'] + workload['vehicles'], 'workload.entities: smaller than declared actors/vehicles') require(isinstance(workload['settings'], dict) and bool(workload['settings']), 'workload.settings: nonempty object required') number(workload['warmup_seconds'], 'workload.warmup_seconds', 30) number(workload['duration_seconds'], 'workload.duration_seconds', 60) keys(run['capture'], ['kind', 'environment', 'tool', 'version'], 'capture') require(run['capture']['kind'] in ('client_frame', 'server_tick'), 'capture.kind: expected client_frame or server_tick') require(run['capture']['environment'] in ('client', 'dedicated_server', 'workbench_headless'), 'capture.environment: unsupported environment') require(run['capture']['kind'] != 'client_frame' or run['capture']['environment'] != 'dedicated_server', 'dedicated server cannot supply client frames') for key in ['tool', 'version']: text(run['capture'][key], f'capture.{key}') def reject_constant(value): raise ValueError(f'Non-finite JSON value: {value}') def unique_object(pairs): result = {} for key, value in pairs: require(key not in result, f'Duplicate JSON field: {key}') result[key] = value return result def compare(manifest_path): manifest_path = Path(manifest_path) raw = manifest_path.read_bytes() data = json.loads(raw, parse_constant=reject_constant, object_pairs_hook=unique_object) finite_json(data) keys(data, ['schema_version', 'sample_origin', 'runs'], 'manifest') require(type(data['schema_version']) is int and data['schema_version'] == 1, 'schema_version: expected 1') require(data['sample_origin'] in ('synthetic', 'measured'), 'sample_origin: expected synthetic or measured') require(isinstance(data['runs'], list) and len(data['runs']) >= 6, 'At least three paired baseline/successor repeats required') paired, paths, reference = {}, set(), None outputs = [] artifacts = {} for run in data['runs']: validate_run(run) comparable = {key: run[key] for key in ['hardware', 'workload', 'capture', 'engine_version']} if reference is None: reference = comparable require(comparable == reference, f"{run['repeat_id']}/{run['variant']}: mismatched hardware, workload, capture settings or engine version") artifact = (run['artifact_id'], sorted(run['addon_ids'])) require(run['variant'] not in artifacts or artifacts[run['variant']] == artifact, 'Artifact/addon set changed between repeats of one variant') artifacts[run['variant']] = artifact pair = paired.setdefault(run['repeat_id'], {}) require(run['variant'] not in pair, f"Duplicate repeat/variant: {run['repeat_id']}/{run['variant']}") path = (manifest_path.parent / run['sample_file']).resolve() require(path not in paths, f'{path}: sample file reused for different runs') paths.add(path) result = {'variant': run['variant'], 'repeat_id': run['repeat_id'], 'artifact_id': run['artifact_id'], 'addon_ids': run['addon_ids'], 'sample_file': run['sample_file'], **load_samples(path, run)} pair[run['variant']] = result outputs.append(result) require(len(paired) >= 3 and all(set(pair) == {'baseline', 'successor'} for pair in paired.values()), 'Every repeat needs exactly one baseline and one successor recording') metric_names = outputs[0]['metrics'] aggregate = {variant: {key: statistics.median([item['metrics'][key] for item in outputs if item['variant'] == variant]) for key in metric_names} for variant in ['baseline', 'successor']} deltas = [] for repeat_id, pair in sorted(paired.items()): baseline, successor = pair['baseline']['metrics'], pair['successor']['metrics'] deltas.append({'repeat_id': repeat_id, 'median_frame_reduction_percent': 100 * (baseline['median_frame_ms'] - successor['median_frame_ms']) / baseline['median_frame_ms'], 'p99_frame_reduction_percent': 100 * (baseline['p99_frame_ms'] - successor['p99_frame_ms']) / baseline['p99_frame_ms'], 'throughput_fps_increase_percent': 100 * (successor['throughput_fps'] - baseline['throughput_fps']) / baseline['throughput_fps']}) return {'schema_version': 1, 'sample_origin': data['sample_origin'], 'acceptance': 'not_evaluated', 'scope': 'Offline arithmetic on caller-supplied recordings and metadata; no authenticity, multiplayer, statistical-significance or release claim. Synthetic samples are not performance evidence. Headless Workbench is not equivalent to ordinary clients.', 'manifest_sha256': hashlib.sha256(raw).hexdigest(), 'matched_conditions': reference, 'percentile_method': 'nearest_rank', 'aggregate_method': 'median of per-run metrics; repeats equally weighted', 'runs': sorted(outputs, key=lambda item: (item['repeat_id'], item['variant'])), 'aggregate': aggregate, 'paired_deltas': deltas, 'median_paired_deltas': {key: statistics.median([delta[key] for delta in deltas]) for key in deltas[0] if key != 'repeat_id'}} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('manifest', type=Path) arguments = parser.parse_args() try: report = compare(arguments.manifest) except (ValueError, OSError, csv.Error) as error: parser.exit(2, f'Performance comparison rejected: {error}\n') print(json.dumps(report, indent=2, allow_nan=False)) if __name__ == '__main__': main()