158 lines
7.3 KiB
Python
158 lines
7.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Read the installed native -logStats format; export coarse records, not frames."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
|
|
MAX_INPUT_BYTES = 64 * 1024 * 1024
|
|
MAX_LINE_BYTES = 16 * 1024
|
|
MAX_RECORDS = 100_000
|
|
MAX_COUNT = (1 << 63) - 1
|
|
_NUMBER = r'([^\s,()]+)'
|
|
_COUNT = r'([0-9]+)'
|
|
_PATTERN = re.compile(
|
|
r'([0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})[ \t]+DEFAULT[ \t]*:[ \t]*'
|
|
r'FPS: ' + _NUMBER + r', frame time \(avg: ' + _NUMBER +
|
|
r' ms, min: ' + _NUMBER + r' ms, max: ' + _NUMBER +
|
|
r' ms, mean: ' + _NUMBER + r' ms, median: ' + _NUMBER +
|
|
r' ms\), Mem: ' + _COUNT + r' kB, Player: ' + _COUNT +
|
|
r', AI: ' + _COUNT + r', AIChar: ' + _COUNT + r', Veh: ' + _COUNT +
|
|
r' \(' + _COUNT + r'\), Proj \(S: ' + _COUNT + r', M: ' + _COUNT +
|
|
r', G: ' + _COUNT + r' \| ' + _COUNT + r'\)' +
|
|
r'(?:, Freezes\(cnt: ' + _COUNT + r', max: ' + _NUMBER + r' ms, len: ' + _COUNT + r'\))?' +
|
|
r', Streaming\(Dynam: ' + _COUNT +
|
|
r', Static: ' + _COUNT + r', Disabled: ' + _COUNT + r' \| New: ' + _COUNT +
|
|
r', Del: ' + _COUNT + r', Bump: ' + _COUNT + r'\)'
|
|
)
|
|
# Only the native DEFAULT category is accepted. A changed/truncated native stats
|
|
# record must fail the export, rather than disappearing among unrelated log lines.
|
|
_CANDIDATE = re.compile(r'\bDEFAULT\b[^\r\n]*(?:\bFPS\b|\bframe time\b)')
|
|
_DECIMAL = re.compile(r'[0-9]+(?:\.[0-9]+)?')
|
|
|
|
|
|
class StatsError(ValueError):
|
|
"""Unsupported or malformed telemetry; no partial successful export."""
|
|
|
|
|
|
def _float(text: str, line: int) -> float:
|
|
if len(text) > 64 or not _DECIMAL.fullmatch(text):
|
|
raise StatsError(f'line {line}: expected finite nonnegative decimal, got {text!r}')
|
|
value = float(text)
|
|
if not math.isfinite(value):
|
|
raise StatsError(f'line {line}: nonfinite telemetry')
|
|
return value
|
|
|
|
|
|
def _count(text: str, line: int) -> int:
|
|
if len(text) > 19 or int(text) > MAX_COUNT:
|
|
raise StatsError(f'line {line}: count exceeds supported integer range')
|
|
return int(text)
|
|
|
|
|
|
def parse_bytes(data: bytes, source: str = '<memory>') -> dict:
|
|
if len(data) > MAX_INPUT_BYTES:
|
|
raise StatsError('input exceeds 64 MiB limit; split logs outside this parser')
|
|
try:
|
|
text = data.decode('utf-8-sig')
|
|
except UnicodeDecodeError as error:
|
|
raise StatsError('input is not UTF-8 text') from error
|
|
records = []
|
|
for line_number, raw_line in enumerate(text.splitlines(), 1):
|
|
if len(raw_line.encode('utf-8')) > MAX_LINE_BYTES:
|
|
raise StatsError(f'line {line_number}: line exceeds 16 KiB limit')
|
|
if not _CANDIDATE.search(raw_line):
|
|
continue
|
|
match = _PATTERN.fullmatch(raw_line)
|
|
if not match:
|
|
raise StatsError(f'line {line_number}: malformed or unsupported native statistics record')
|
|
fields = match.groups()
|
|
timestamp = fields[0]
|
|
hour, minute, second = map(int, (timestamp[:2], timestamp[3:5], timestamp[6:8]))
|
|
if hour > 23 or minute > 59 or second > 59:
|
|
raise StatsError(f'line {line_number}: invalid native clock timestamp')
|
|
numbers = [_float(value, line_number) for value in fields[1:7]]
|
|
fps, avg, minimum, maximum, mean, median = numbers
|
|
if minimum > maximum or any(value < minimum or value > maximum for value in (avg, mean, median)):
|
|
raise StatsError(f'line {line_number}: inconsistent frame-time summary bounds')
|
|
counts = [_count(value, line_number) for value in fields[7:17]]
|
|
memory, player, ai, ai_char, vehicle, vehicle_parenthesized, s, m, g, projectile_trailing = counts
|
|
freezes = None
|
|
if fields[17] is not None:
|
|
freezes = {'cnt': _count(fields[17], line_number),
|
|
'max_ms': _float(fields[18], line_number),
|
|
'len': _count(fields[19], line_number)}
|
|
dynamic, static, disabled, new, deleted, bump = [_count(value, line_number) for value in fields[20:]]
|
|
records.append({
|
|
'line_number': line_number,
|
|
'timestamp': timestamp,
|
|
'window_duration_ms': None,
|
|
'fps': fps,
|
|
'frame_time_summary_ms': dict(avg=avg, min=minimum, max=maximum, mean=mean, median=median),
|
|
'memory_kB': memory,
|
|
'native_populations': {'Player': player, 'AI': ai, 'AIChar': ai_char,
|
|
'Veh': {'value': vehicle, 'parenthesized': vehicle_parenthesized}},
|
|
'native_projectiles': {'S': s, 'M': m, 'G': g, 'trailing': projectile_trailing},
|
|
'native_freezes': freezes,
|
|
'native_streaming': {'Dynam': dynamic, 'Static': static, 'Disabled': disabled,
|
|
'New': new, 'Del': deleted, 'Bump': bump},
|
|
'ai_only_duration_ms': None,
|
|
'raw_line': raw_line,
|
|
})
|
|
if len(records) > MAX_RECORDS:
|
|
raise StatsError('statistics record limit exceeded')
|
|
if not records:
|
|
raise StatsError('no supported native statistics records found')
|
|
return {
|
|
'schema_version': 1,
|
|
'kind': 'native-server-coarse-statistics',
|
|
'source': {'path': source, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data)},
|
|
'performance_acceptance': 'not_evaluated',
|
|
'unavailable': {
|
|
'ai_only_duration_ms': 'Native -logStats records contain no AI-only execution duration.',
|
|
'raw_frame_samples': 'Each record contains coarse summaries, not individual frames.',
|
|
'percentiles': 'Slow-tail percentiles cannot be recovered from these summaries.',
|
|
'window_duration_ms': 'The line does not identify its exact sampling window.',
|
|
'calendar_date': 'Record timestamps contain only the native clock time; no date inferred.',
|
|
},
|
|
'limitations': [
|
|
'Native labels are preserved; AI and AIChar are not equated or treated as AI execution time.',
|
|
'Vehicle parenthesized and projectile trailing numbers are retained without inferred scope.',
|
|
'Streaming counts are not treated as a complete loaded-world entity census.',
|
|
'An absent Freezes block is null, not zero; cnt and len retain native labels without inferred duration or frame counts.',
|
|
'Records retain source order, including clock rollover; no elapsed intervals or missing windows are invented.',
|
|
'Parser success does not check engine/script errors, source provenance, equivalent workloads or release qualification.',
|
|
'This output is not compare_performance.py v1 frame CSV input.',
|
|
],
|
|
'record_count': len(records),
|
|
'records': records,
|
|
}
|
|
|
|
|
|
def parse_file(path: Path) -> dict:
|
|
with path.open('rb') as handle:
|
|
data = handle.read(MAX_INPUT_BYTES + 1)
|
|
return parse_bytes(data, str(path.resolve()))
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('log', type=Path, help='One existing UTF-8 native console log; read only')
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
result = parse_file(args.log)
|
|
except (OSError, StatsError) as error:
|
|
print(f'native statistics export refused: {error}', file=sys.stderr)
|
|
return 1
|
|
print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|