228 lines
12 KiB
Python
228 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare declared equivalent frame/tick recordings; never grants acceptance."""
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import statistics
|
|
|
|
COLUMNS = ['frame_index', 'elapsed_ms', 'frame_time_ms', 'players', 'ai', 'vehicles', 'entities', 'ai_simulation_ms']
|
|
POPULATION = ['players', 'ai', 'vehicles', 'entities']
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise ValueError(message)
|
|
|
|
|
|
def keys(value, expected, label):
|
|
require(isinstance(value, dict) and set(value) == set(expected), f'{label}: expected fields {expected}')
|
|
|
|
|
|
def number(value, label, minimum=0):
|
|
require(type(value) in (float, int) and math.isfinite(value) and value >= minimum, f'{label}: invalid number')
|
|
return value
|
|
|
|
|
|
def text(value, label):
|
|
require(isinstance(value, str) and bool(value.strip()), f'{label}: required nonempty string')
|
|
|
|
|
|
def integer(value, label, minimum=0):
|
|
require(type(value) is int and value >= minimum, f'{label}: invalid integer')
|
|
|
|
|
|
def finite_json(value):
|
|
if isinstance(value, float):
|
|
require(math.isfinite(value), 'Non-finite JSON number')
|
|
elif isinstance(value, dict):
|
|
for item in value.values():
|
|
finite_json(item)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
finite_json(item)
|
|
|
|
|
|
def percentile(values, fraction):
|
|
"""Nearest rank, not interpolation; slow-tail frame times use upper ranks."""
|
|
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
|
|
|
|
|
|
def metrics(frames, ai_load):
|
|
fps = [1000 / value for value in frames]
|
|
result = {
|
|
'median_frame_ms': statistics.median(frames),
|
|
'p95_frame_ms': percentile(frames, .95),
|
|
'p99_frame_ms': percentile(frames, .99),
|
|
'median_fps': statistics.median(fps),
|
|
'p05_fps': percentile(fps, .05),
|
|
'p01_fps': percentile(fps, .01),
|
|
'throughput_fps': 1000 * len(frames) / sum(frames),
|
|
'median_ai_simulation_ms': statistics.median(ai_load),
|
|
'p95_ai_simulation_ms': percentile(ai_load, .95),
|
|
}
|
|
finite_json(result)
|
|
return result
|
|
|
|
|
|
def load_samples(path, run):
|
|
workload = run['workload']
|
|
warmup = workload['warmup_seconds'] * 1000
|
|
end = warmup + workload['duration_seconds'] * 1000
|
|
frames, ai_load = [], []
|
|
boundary_frames = []
|
|
previous = 0.0
|
|
total_frame_ms = 0.0
|
|
row_count = 0
|
|
with path.open(newline='', encoding='utf-8') as stream:
|
|
reader = csv.DictReader(stream)
|
|
require(reader.fieldnames == COLUMNS, f'{path}: expected exact CSV header {COLUMNS}')
|
|
for index, row in enumerate(reader, 1):
|
|
label = f'{path}: row {index + 1}'
|
|
require(set(row) == set(COLUMNS) and all(v is not None for v in row.values()), f'{label}: incomplete/extra columns')
|
|
try:
|
|
frame_index = int(row['frame_index'])
|
|
elapsed = float(row['elapsed_ms'])
|
|
frame = float(row['frame_time_ms'])
|
|
ai_ms = float(row['ai_simulation_ms'])
|
|
population = {key: int(row[key]) for key in POPULATION}
|
|
except (ValueError, TypeError) as error:
|
|
raise ValueError(f'{label}: invalid numeric sample') from error
|
|
require(frame_index == index, f'{label}: missing/duplicate/out-of-order frame index')
|
|
require(all(math.isfinite(v) for v in (elapsed, frame, ai_ms)) and frame >= .001 and elapsed > previous and ai_ms >= 0, f'{label}: invalid frame/time/load')
|
|
require(abs((elapsed - previous) - frame) <= .1, f'{label}: elapsed delta differs from frame time by over 0.1 ms')
|
|
total_frame_ms += frame
|
|
require(abs(total_frame_ms - elapsed) <= .1, f'{label}: cumulative elapsed time differs from recorded frames')
|
|
require(previous < end, f'{label}: samples continue beyond the declared capture window')
|
|
for key in POPULATION:
|
|
require(population[key] == workload[key], f'{label}: {key} workload changed')
|
|
if previous >= warmup and elapsed <= end:
|
|
frames.append(frame)
|
|
ai_load.append(ai_ms)
|
|
else:
|
|
overlap = max(0, min(elapsed, end) - max(previous, warmup))
|
|
if overlap > 0:
|
|
boundary_frames.append({'frame_index': frame_index,
|
|
'frame_time_ms': frame,
|
|
'measurement_overlap_ms': overlap})
|
|
previous = elapsed
|
|
row_count += 1
|
|
require(previous >= end, f'{path}: incomplete recording; warmup plus measurement duration not covered')
|
|
measured_ms = sum(frames)
|
|
coverage = measured_ms / (workload['duration_seconds'] * 1000)
|
|
require(coverage >= .99, f'{path}: only {coverage:.3%} of declared measurement duration has complete frames; boundary stalls/intervals exclude over 1%')
|
|
require(len(frames) >= 120, f'{path}: need at least 120 complete measurement frames/ticks')
|
|
return {'samples_total': row_count, 'samples_measured': len(frames),
|
|
'measured_frame_seconds': measured_ms / 1000,
|
|
'measurement_coverage_fraction': coverage,
|
|
'boundary_frames_excluded': boundary_frames,
|
|
'csv_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'metrics': metrics(frames, ai_load)}
|
|
|
|
|
|
def validate_run(run):
|
|
keys(run, ['variant', 'repeat_id', 'artifact_id', 'addon_ids', 'engine_version', 'hardware', 'workload', 'capture', 'sample_file'], 'run')
|
|
require(run['variant'] in ('baseline', 'successor'), 'run: variant must be baseline or successor')
|
|
for key in ['repeat_id', 'artifact_id', 'engine_version', 'sample_file']:
|
|
text(run[key], key)
|
|
require(isinstance(run['addon_ids'], list) and bool(run['addon_ids']), 'addon_ids: nonempty list required')
|
|
for addon in run['addon_ids']:
|
|
text(addon, 'addon id')
|
|
require(len(set(run['addon_ids'])) == len(run['addon_ids']), 'addon_ids: duplicates')
|
|
keys(run['hardware'], ['cpu', 'gpu', 'ram_gib', 'os', 'driver'], 'hardware')
|
|
for key in ['cpu', 'gpu', 'os', 'driver']:
|
|
text(run['hardware'][key], f'hardware.{key}')
|
|
number(run['hardware']['ram_gib'], 'hardware.ram_gib', 1)
|
|
workload = run['workload']
|
|
keys(workload, ['map', 'route', *POPULATION, 'settings', 'warmup_seconds', 'duration_seconds'], 'workload')
|
|
text(workload['map'], 'workload.map')
|
|
text(workload['route'], 'workload.route')
|
|
for key in POPULATION:
|
|
integer(workload[key], f'workload.{key}', 1 if key == 'players' else 0)
|
|
require(workload['entities'] >= workload['players'] + workload['ai'] + workload['vehicles'], 'workload.entities: smaller than declared actors/vehicles')
|
|
require(isinstance(workload['settings'], dict) and bool(workload['settings']), 'workload.settings: nonempty object required')
|
|
number(workload['warmup_seconds'], 'workload.warmup_seconds', 30)
|
|
number(workload['duration_seconds'], 'workload.duration_seconds', 60)
|
|
keys(run['capture'], ['kind', 'environment', 'tool', 'version'], 'capture')
|
|
require(run['capture']['kind'] in ('client_frame', 'server_tick'), 'capture.kind: expected client_frame or server_tick')
|
|
require(run['capture']['environment'] in ('client', 'dedicated_server', 'workbench_headless'), 'capture.environment: unsupported environment')
|
|
require(run['capture']['kind'] != 'client_frame' or run['capture']['environment'] != 'dedicated_server', 'dedicated server cannot supply client frames')
|
|
for key in ['tool', 'version']:
|
|
text(run['capture'][key], f'capture.{key}')
|
|
|
|
|
|
def reject_constant(value):
|
|
raise ValueError(f'Non-finite JSON value: {value}')
|
|
|
|
|
|
def unique_object(pairs):
|
|
result = {}
|
|
for key, value in pairs:
|
|
require(key not in result, f'Duplicate JSON field: {key}')
|
|
result[key] = value
|
|
return result
|
|
|
|
|
|
def compare(manifest_path):
|
|
manifest_path = Path(manifest_path)
|
|
raw = manifest_path.read_bytes()
|
|
data = json.loads(raw, parse_constant=reject_constant, object_pairs_hook=unique_object)
|
|
finite_json(data)
|
|
keys(data, ['schema_version', 'sample_origin', 'runs'], 'manifest')
|
|
require(type(data['schema_version']) is int and data['schema_version'] == 1, 'schema_version: expected 1')
|
|
require(data['sample_origin'] in ('synthetic', 'measured'), 'sample_origin: expected synthetic or measured')
|
|
require(isinstance(data['runs'], list) and len(data['runs']) >= 6, 'At least three paired baseline/successor repeats required')
|
|
paired, paths, reference = {}, set(), None
|
|
outputs = []
|
|
artifacts = {}
|
|
for run in data['runs']:
|
|
validate_run(run)
|
|
comparable = {key: run[key] for key in ['hardware', 'workload', 'capture', 'engine_version']}
|
|
if reference is None:
|
|
reference = comparable
|
|
require(comparable == reference, f"{run['repeat_id']}/{run['variant']}: mismatched hardware, workload, capture settings or engine version")
|
|
artifact = (run['artifact_id'], sorted(run['addon_ids']))
|
|
require(run['variant'] not in artifacts or artifacts[run['variant']] == artifact, 'Artifact/addon set changed between repeats of one variant')
|
|
artifacts[run['variant']] = artifact
|
|
pair = paired.setdefault(run['repeat_id'], {})
|
|
require(run['variant'] not in pair, f"Duplicate repeat/variant: {run['repeat_id']}/{run['variant']}")
|
|
path = (manifest_path.parent / run['sample_file']).resolve()
|
|
require(path not in paths, f'{path}: sample file reused for different runs')
|
|
paths.add(path)
|
|
result = {'variant': run['variant'], 'repeat_id': run['repeat_id'], 'artifact_id': run['artifact_id'], 'addon_ids': run['addon_ids'], 'sample_file': run['sample_file'], **load_samples(path, run)}
|
|
pair[run['variant']] = result
|
|
outputs.append(result)
|
|
require(len(paired) >= 3 and all(set(pair) == {'baseline', 'successor'} for pair in paired.values()), 'Every repeat needs exactly one baseline and one successor recording')
|
|
metric_names = outputs[0]['metrics']
|
|
aggregate = {variant: {key: statistics.median([item['metrics'][key] for item in outputs if item['variant'] == variant]) for key in metric_names} for variant in ['baseline', 'successor']}
|
|
deltas = []
|
|
for repeat_id, pair in sorted(paired.items()):
|
|
baseline, successor = pair['baseline']['metrics'], pair['successor']['metrics']
|
|
deltas.append({'repeat_id': repeat_id,
|
|
'median_frame_reduction_percent': 100 * (baseline['median_frame_ms'] - successor['median_frame_ms']) / baseline['median_frame_ms'],
|
|
'p99_frame_reduction_percent': 100 * (baseline['p99_frame_ms'] - successor['p99_frame_ms']) / baseline['p99_frame_ms'],
|
|
'throughput_fps_increase_percent': 100 * (successor['throughput_fps'] - baseline['throughput_fps']) / baseline['throughput_fps']})
|
|
return {'schema_version': 1, 'sample_origin': data['sample_origin'], 'acceptance': 'not_evaluated',
|
|
'scope': 'Offline arithmetic on caller-supplied recordings and metadata; no authenticity, multiplayer, statistical-significance or release claim. Synthetic samples are not performance evidence. Headless Workbench is not equivalent to ordinary clients.',
|
|
'manifest_sha256': hashlib.sha256(raw).hexdigest(), 'matched_conditions': reference,
|
|
'percentile_method': 'nearest_rank', 'aggregate_method': 'median of per-run metrics; repeats equally weighted',
|
|
'runs': sorted(outputs, key=lambda item: (item['repeat_id'], item['variant'])),
|
|
'aggregate': aggregate, 'paired_deltas': deltas,
|
|
'median_paired_deltas': {key: statistics.median([delta[key] for delta in deltas]) for key in deltas[0] if key != 'repeat_id'}}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('manifest', type=Path)
|
|
arguments = parser.parse_args()
|
|
try:
|
|
report = compare(arguments.manifest)
|
|
except (ValueError, OSError, csv.Error) as error:
|
|
parser.exit(2, f'Performance comparison rejected: {error}\n')
|
|
print(json.dumps(report, indent=2, allow_nan=False))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|