Count occupied restarts as failed missions; sync published Workshop version
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-11-07 01:05:43 +01:00
parent 161100cf08
commit e421dcfc15
14 changed files with 640 additions and 50 deletions
@@ -0,0 +1,63 @@
#ifdef WORKBENCH
// Disposable native fixture only. Keep native serialization, profile storage,
// PlayerManager occupancy and production constructor/Finish; replace transport.
modded class ERR_Telemetry
{
protected override void Pulse() {}
bool ERR_TestCheck(bool valid, string detail)
{
if (valid) Print("ERR SMOKE PASS: telemetry " + detail);
else Print("ERR SMOKE FAIL: telemetry " + detail, LogLevel.ERROR);
return valid;
}
int ERR_TestFinalCount(string mission)
{
int count;
foreach (string payload : m_State.pending)
{
JsonLoadContext json = new JsonLoadContext(); string id, type, outcome; int phase;
if (!json.LoadFromString(payload)) return -1;
json.ReadValue("missionId", id); json.ReadValue("type", type);
if (id != mission || type != "mission_finished") continue;
json.ReadValue("outcome", outcome); json.ReadValue("phase", phase);
if (outcome != "defeat" || phase != ERR_Phase.DEFEAT) return -1;
count++;
}
return count;
}
void ERR_TestFreeze()
{
m_Closed = true;
GetGame().GetCallqueue().Remove(Pulse);
}
bool ERR_TestOccupiedRecovery()
{
int connected = GetGame().GetPlayerManager().GetPlayerCount();
PrintFormat("ERR TELEMETRY NATIVE: connected=%1 mission=%2", connected, m_MissionId);
if (!ERR_TestCheck(Enabled() && connected > 0, "uses actual connected native player")) return false;
Checkpoint();
ERR_TelemetryState disk;
if (!ERR_TestCheck(ReadState(STATE, disk) && !disk.restartFailure.IsEmpty() && disk.previousOutcome == "unfinished", "occupied checkpoint round-trips through native JSON and profile storage")) return false;
if (!ERR_TestCheck(ERR_TestFinalCount(m_MissionId) == 0, "live checkpoint is not queued as a failure")) return false;
// Constructor recovery from persisted unfinished state, as after a forced stop.
ref ERR_Telemetry recovered = new ERR_Telemetry(m_Mode);
bool recoveredOK = recovered.ERR_TestFinalCount(m_MissionId) == 1;
recovered.ERR_TestFreeze();
if (!ERR_TestCheck(recoveredOK, "fresh service recovers exactly one defeat for the old mission")) return false;
// Restore the live service's state for the real EndGameMode test below.
if (!ERR_TestCheck(Save(), "restore original live checkpoint for native shutdown")) return false;
return true;
}
bool ERR_TestFinished()
{
ERR_TelemetryState disk;
if (!ERR_TestCheck(m_Ended && m_State.previousOutcome == "defeat" && m_State.restartFailure.IsEmpty() && ERR_TestFinalCount(m_MissionId) == 1, "actual native end records exactly one occupied failure")) return false;
if (!ERR_TestCheck(ReadState(STATE, disk) && disk.previousOutcome == "defeat" && disk.restartFailure.IsEmpty(), "completed result persisted and candidate cleared")) return false;
ref ERR_Telemetry recovered = new ERR_Telemetry(m_Mode);
bool recoveredOK = recovered.ERR_TestFinalCount(m_MissionId) == 1;
recovered.ERR_TestFreeze();
if (!ERR_TestCheck(recoveredOK, "reopening completed state does not duplicate the failure")) return false;
return Save();
}
}
#endif
+43
View File
@@ -0,0 +1,43 @@
# Native telemetry restart test
Run from the repository root with a disposable existing Proton test prefix and
the installed authorized dependency tree:
```sh
python3 tests/native/telemetry_restart/run.py \
--work .local/restart-failure/verified \
--compat .local/rhs-19-work/compat \
--external-addons '/home/jakub/.local/share/Steam/steamapps/compatdata/1874880/pfx/drive_c/users/steamuser/Documents/My Games/ArmaReforger/addons'
```
The work directory must be new. The runner copies the current addon and adds the
fixture only to that copy. It reuses the native preparation-stage shutdown probe
so extraction placement is not a prerequisite. It then explicitly stops and
relaunches its owned Workbench process with the same
private profile. Workbench hosting does not automatically reload this fixture.
Two rounds use the actual native PlayerManager connection count and EndGameMode
callback. Production reload behavior is not changed.
The fixture checks native JSON/profile round-trips, an occupied checkpoint not
being submitted while the round is live, constructor recovery from unfinished
state, a single persisted defeat on shutdown, and completed-state reopening
without another defeat. Forced-stop recovery is simulated by constructing a new
service from the saved unfinished checkpoint; this is not an AMP process-kill
or multiplayer dedicated-server test. Empty occupancy and completion variants
also have production-method tests in `tests/test_telemetry.py`.
The copied test class replaces only the telemetry dispatch tick, preventing
network submissions. A reserved `.invalid` endpoint is an additional safeguard.
The fixture restores the original service's state after each recovery probe.
No test code or test results are included in the published addon.
Native evidence is retained under the work directory's `profile/logs`, including
source hashes and cleanup verification. Queued payloads are in
`profile/profile/EscapistsTelemetryState.json` and can be checked with the
receiver's `validateEvent` function.
The general native report also grades optional asset and backend errors. This
fixture retains that complete report as `telemetry-check.json`, while its result
requires completed telemetry assertions, no script/runtime failures, unchanged
inputs, confirmed process cleanup and valid queued results. It does not certify
Rally/RHS/WCS assets, DLC ownership or the rest of the mission.
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Run isolated native occupancy, JSON persistence, recovery and shutdown checks."""
import argparse
import json
from pathlib import Path
import shutil
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[3]
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--work', type=Path, required=True, help='New disposable directory; must not exist')
p.add_argument('--compat', type=Path, required=True, help='Existing private test Proton prefix')
p.add_argument('--external-addons', type=Path, required=True)
a = p.parse_args()
work = a.work.resolve()
if work.exists():
p.error('--work must not exist; preserve evidence from earlier runs')
work.mkdir(parents=True)
source, profile = work/'source', work/'profile'
for name in ['addons', 'tools']:
shutil.copytree(ROOT/name, source/name)
(source/'docs').mkdir()
shutil.copy2(ROOT/'docs/external-addons.json', source/'docs/external-addons.json')
def replace(path, before, after):
text = path.read_text()
if text.count(before) != 1:
raise ValueError(f'Fixture insertion point changed in {path.name}')
path.write_text(text.replace(before, after))
scripts = source/'addons/core/Scripts/Game/ERR/00_Core'
shutil.copy2(Path(__file__).with_name('ERR_TelemetryCheckpointTest.c'), scripts/'ERR_TelemetryCheckpointTest.c')
# Modded classes can only access fields/methods of earlier layers. Test accessors
# follow telemetry; the existing native-end fixture is moved after that layer.
replace(scripts/'ERR_TelemetryLifecycle.c', ' void ERR_RefreshTelemetry()',
' bool ERR_TestTelemetryBefore() { return m_ERRTelemetry && m_ERRTelemetry.ERR_TestOccupiedRecovery(); }\n'
' bool ERR_TestTelemetryAfter() { return m_ERRTelemetry && m_ERRTelemetry.ERR_TestFinished(); }\n'
' void ERR_RefreshTelemetry()')
probe = scripts/'ERR_NativeEndProbe.c'
replace(probe, 'm_ERREndData = SCR_GameModeEndData.CreateSimple',
'if (!ERR_EndCheck(ERR_TestTelemetryBefore(), "occupied telemetry snapshot and recovery")) return;\n'
' m_ERREndData = SCR_GameModeEndData.CreateSimple')
replace(probe, 'EndGameMode(m_ERREndData);',
'EndGameMode(m_ERREndData);\n'
' if (!ERR_EndCheck(ERR_TestTelemetryAfter(), "native shutdown records one defeat")) return;')
probe.rename(scripts/'ERR_ZTelemetryNativeEndProbe.c')
runner = source/'tools/headless_workbench.py'
replace(runner, 'if a.roadblock_probe_count is not None: command',
"command += ['-errTelemetryTest', '1']\nif a.roadblock_probe_count is not None: command")
# Use two explicit process launches: Workbench need not auto-reload a hosted game.
replace(runner, "a.project != 'core' or a.rounds != 2 or a.timeout < 120", "a.project != 'core' or a.rounds != 1 or a.timeout < 120")
replace(runner, "if any((profile/name).exists() or (profile/name).is_symlink() for name in fixture_names): p.error('native-end fixture paths must not preexist; preserve prior evidence before retrying')",
'# This isolated fixture intentionally preserves the first instance marker.')
(profile/'profile').mkdir(parents=True)
(profile/'profile/EscapistsTelemetry.json').write_text(
'{"endpoint":"https://telemetry.invalid/api/mission-events",'
'"token":"local-test-only","serverName":"Isolated restart test"}\n')
command = [sys.executable, str(runner), '--compat', str(a.compat.resolve()),
'--profile', str(profile), '--external-addons', str(a.external_addons.resolve()),
'--smoke', 'native-end-preparing', '--project', 'core', '--rounds', '1', '--timeout', '120']
sys.path.insert(0, str(ROOT/'tools'))
from native_report import analyze
for _ in range(2):
old_logs = set((profile/'logs').glob('logs_*'))
result = subprocess.run(command, check=False)
new_logs = set((profile/'logs').glob('logs_*')) - old_logs
assert len(new_logs) == 1, 'Expected exactly one fresh native session'
session = new_logs.pop()
evidence = json.loads((session/'runner.json').read_text())
report = analyze((session/'console.log').read_text(errors='replace'), session/'console.log')
assert result.returncode in (0, 1)
assert evidence['result'] == 'smoke-completed' and evidence['cleanup_confirmed']
assert evidence['source_unchanged_after_run'] and evidence['external_addons_unchanged_after_run']
assert report['smoke_completions'] == 1 and not report['script_or_runtime_failures']
assert not report['missing_end_screen']
# The generic runner also grades all optional asset/backend errors. Preserve
# its complete report; this fixture grades telemetry, not DLC/asset integrity.
report['telemetry_assertions_passed'] = True
(session/'telemetry-check.json').write_text(json.dumps(report, indent=2)+'\n')
if report['engine_errors']:
print(f"Telemetry checks passed; {len(report['engine_errors'])} asset/backend errors retained in {session/'telemetry-check.json'}", flush=True)
state_path = profile/'profile/EscapistsTelemetryState.json'
events = [json.loads(payload) for payload in json.loads(state_path.read_text())['state']['pending']]
finished = [event for event in events if event['type'] == 'mission_finished']
assert len(finished) == 2 and len({event['missionId'] for event in finished}) == 2
assert all(event['outcome'] == 'defeat' and event['phase'] == 6 and event['connectedPlayers'] > 0 for event in finished)
restarts = [event for event in events if event['type'] == 'mission_restarted']
assert len(restarts) == 1 and restarts[0]['previousOutcome'] == 'defeat'
assert restarts[0]['previousMissionId'] in {event['missionId'] for event in finished}
subprocess.run(['node', '--input-type=module', '-e',
"import {readFileSync} from 'node:fs'; import {validateEvent} from "
+ json.dumps((ROOT/'services/telemetry-worker/src/validation.js').as_uri()) + ";"
+ "const {state}=JSON.parse(readFileSync(process.argv[1],'utf8'));"
+ "for(const payload of state.pending) validateEvent(JSON.parse(payload));", str(state_path)], check=True)
print('PASS: two native process instances, real occupied shutdowns, durable recovery, no duplicate results and receiver-valid native payloads')
+38 -7
View File
@@ -3,10 +3,10 @@ from policy_harness import ROOT, body, run
p=ROOT/'addons/core/Scripts/Game/ERR/00_Core/ERR_Telemetry.c'
s=p.read_text()
def cpp(v):
for a,b in [('protected ',''),('m_Config.', 'config.'),('Replication.IsServer()', 'authority'),('m_Mode.Settings()', 'settings'),('FileIO.DeleteFile(', 'DeleteFile('),('ERR_Phase.','Phase::'),('ERR_TelemetryRules.','Rules::'),('ERR_MissionRules.','MissionRules::'),('Math.Min(', 'std::min('),('Math.Pow(', 'power('),('System.GetTickCount()', '::now'),('LogLevel.WARNING','0'),('m_State.', 'm_State->'),('m_Callback = null', 'm_Callback = nullptr'),('m_Context = null', 'm_Context = nullptr'),('m_Mode = null', 'm_Mode = nullptr'),('RestCallback callback', 'RestCallback* callback'),('callback.GetHttpCode()', 'callback->GetHttpCode()'),('GetGame().GetCallqueue().Remove(Pulse)', 'removed++')]: v=v.replace(a,b)
for a,b in [('protected ',''),('JsonLoadContext queued = new JsonLoadContext();', 'JsonLoadContext queued;'),('GetGame().GetPlayerManager().GetPlayerCount()', 'connections'),('m_Mode.Phase()', 'phase'),('m_Config.', 'config.'),('Replication.IsServer()', 'authority'),('m_Mode.Settings()', 'settings'),('FileIO.DeleteFile(', 'DeleteFile('),('ERR_Phase.','Phase::'),('ERR_TelemetryRules.','Rules::'),('ERR_MissionRules.','MissionRules::'),('Math.Min(', 'std::min('),('Math.Pow(', 'power('),('System.GetTickCount()', '::now'),('LogLevel.WARNING','0'),('m_State.', 'm_State->'),('m_Callback = null', 'm_Callback = nullptr'),('m_Context = null', 'm_Context = nullptr'),('m_Mode = null', 'm_Mode = nullptr'),('RestCallback callback', 'RestCallback* callback'),('callback.GetHttpCode()', 'callback->GetHttpCode()'),('GetGame().GetCallqueue().Remove(Pulse)', 'removed++')]: v=v.replace(a,b)
return v
rules='\n'.join(cpp(body(p,x)) for x in ['static bool Acknowledged(', 'static int RetrySeconds(', 'static string Outcome('])
methods='\n'.join(cpp(body(p,x)) for x in ['bool Enabled(', 'void Refresh(', 'protected void Purge(', 'void Finish(', 'void PhaseChanged(', 'protected void Delivered(', 'protected void Failed(', 'void Close('])
methods='\n'.join(cpp(body(p,x)) for x in ['bool Enabled(', 'void Refresh(', 'protected void Purge(', 'void Finish(', 'protected void RecoverRestart(', 'protected void Enqueue(', 'protected void Checkpoint(', 'void RosterChanged(', 'void PhaseChanged(', 'protected void Delivered(', 'protected void Failed(', 'void Close('])
methods=methods.replace('GetGame().GetCallqueue().CallLater(Pulse, 1, false)', 'pulses++')
run(r'''
#include <cassert>
@@ -21,15 +21,16 @@ namespace MissionRules {bool Terminal(int p){return p==Phase::VICTORY||p==Phase:
struct Rules {
'''+rules+r'''
};
struct Pending:std::vector<string>{using std::vector<string>::operator=;void Clear(){clear();}bool IsEmpty(){return empty();}void RemoveItem(string x){erase(std::remove(begin(),end(),x),end());}};
struct State{string previousOutcome,previousMissionId;Pending pending;};
struct Pending:std::vector<string>{using std::vector<string>::operator=;int Count(){return size();}void Insert(string x){push_back(x);}void InsertAt(string x,int at){insert(begin()+at,x);}void Remove(int at){erase(begin()+at);}void Clear(){clear();}bool IsEmpty(){return empty();}void RemoveItem(string x){erase(std::remove(begin(),end(),x),end());}};
struct JsonLoadContext{string value;bool LoadFromString(string x){value=x;return true;}bool ReadValue(string key,string& type){type=value;return key=="type";}};
struct State{int droppedEvents=0;string previousOutcome,previousMissionId,restartFailure;Pending pending;};
struct RestCallback{int code=200;int GetHttpCode(){return code;}};
struct Service {
struct Config{string Endpoint="https://fixture.invalid/";}config;
struct Settings{int enabled=1;int Get(string){return enabled;}explicit operator bool()const{return true;}}settings;
Pending m_Players;bool authority=true,m_Collected=false,m_Partial=false,m_TruncatedPlayers=false,current=true;
string m_MissionId="same-round";string STATE="queue";
int m_Started=now,m_LastSample=0,m_NextHeartbeat=0,m_Peak=0,deletes=0;
int connections=0,phase=Phase::PRISON;int m_Started=now,m_LastSample=0,m_NextHeartbeat=0,m_Peak=0,deletes=0;
void DeleteFile(std::string){deletes++;}
bool m_Enabled=true,m_Closed=false,m_Ended=false,m_Dirty=false,desired=true,saveOK=true;
State state;State* m_State=&state;RestCallback* m_Callback=nullptr;void* m_Context=nullptr;void* m_Mode=nullptr;
@@ -38,7 +39,10 @@ struct Service {
int pulses=0;void UpdateCommunity(RestCallback*){}void CommunityUnavailable(){}
bool Current(){return current;}
void Sample(){samples++;}
void Event(string t,string o="",string p="",string prior=""){type=t;result=o;events++;}
void Event(string t,string o="",string p="",string prior="",bool checkpoint=false){
if(checkpoint){state.restartFailure="snapshot";Save();return;}type=t;result=o;events++;
}
bool Save(){saves++;if(saveOK)m_Dirty=false;return saveOK;}
'''+methods+r'''
};
@@ -59,6 +63,33 @@ int main(){
Service loss;loss.Finish("preparation_failed");loss.Close();loss.Close();
assert(loss.events==1&&loss.result=="preparation_failed"&&loss.removed==1);
Service abandoned;abandoned.Close();assert(abandoned.result=="interrupted");
Service occupied;occupied.connections=2;occupied.Close();
assert(occupied.events==1&&occupied.result=="defeat"&&occupied.state.previousOutcome=="defeat");
occupied.Close();assert(occupied.events==1);
Service emptyRestart;emptyRestart.Finish("restarted_empty");emptyRestart.Close();assert(emptyRestart.result=="restarted_empty");
Service checkpoint;checkpoint.connections=1;checkpoint.RosterChanged(1);
assert(checkpoint.state.restartFailure=="snapshot"&&checkpoint.events==0&&checkpoint.saves==1);
checkpoint.connections=0;checkpoint.RosterChanged(1);
assert(checkpoint.state.restartFailure.IsEmpty()&&checkpoint.events==0&&checkpoint.saves==2);
checkpoint.connections=1;checkpoint.Checkpoint();checkpoint.Finish("victory");checkpoint.Checkpoint();
assert(checkpoint.state.restartFailure.IsEmpty()&&checkpoint.result=="victory");
Service crash;crash.m_Enabled=false;crash.state.previousMissionId="crashed-round";
crash.state.previousOutcome="unfinished";crash.state.restartFailure="persisted-event-with-stable-id";
crash.Refresh();assert(crash.state.pending.size()==1&&crash.state.pending[0]=="persisted-event-with-stable-id");
assert(crash.state.restartFailure.IsEmpty()&&crash.state.previousMissionId=="same-round");
crash.RecoverRestart();assert(crash.state.pending.size()==1);
Service full;full.m_InFlight="in-flight";full.state.pending.push_back("in-flight");
for(int i=1;i<64;i++)full.state.pending.push_back(i==10?"heartbeat":"old");
full.state.previousOutcome="unfinished";full.state.restartFailure="recovered";full.RecoverRestart();
assert(full.state.pending.size()==64&&full.state.pending[0]=="recovered"&&full.state.pending[1]=="in-flight");
assert(std::count(full.state.pending.begin(),full.state.pending.end(),"heartbeat")==0&&full.state.droppedEvents==1);
Service legacy;legacy.state.previousOutcome="unfinished";legacy.RecoverRestart();assert(legacy.state.pending.empty());
for(string result:{"victory","defeat","preparation_failed","restarted_empty","interrupted"}){
Service completed;completed.state.previousOutcome=result;completed.state.restartFailure="stale";completed.RecoverRestart();
assert(completed.state.pending.empty()&&completed.state.previousOutcome==result&&completed.state.restartFailure.IsEmpty());
}
Service optout;optout.state.restartFailure="private-data";optout.state.previousMissionId="old";
optout.settings.enabled=0;optout.Refresh();assert(optout.state.restartFailure.IsEmpty()&&optout.state.pending.empty());
Service disabled;disabled.settings.enabled=0;disabled.Finish("victory");assert(disabled.events==0);
RestCallback old,current;Service delivery;
delivery.state.pending={"first","second"};delivery.m_InFlight="first";delivery.m_Callback=&current;
@@ -94,4 +125,4 @@ assert 'ReadValue("token"' in s and 'WriteValue("token", m_Config.Token)' in s
event=body(p,'protected void Event(')
assert 'Token' not in event and 'Endpoint' not in event
assert 'DidEscape' in body(p,'protected void Sample(')
print('PASS: telemetry outcomes, exactly-once finish, opt-out gate, stale callbacks, HTTP status handling and bounded retry')
print('PASS: telemetry outcomes, occupied/empty restarts, durable recovery, bounded queue, opt-out, exactly-once finish and HTTP retries')
+45
View File
@@ -2,6 +2,9 @@
import importlib.util
from pathlib import Path
import unittest
import json
import io
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location('workshop_changelog', ROOT / 'tools/workshop_changelog.py')
@@ -40,5 +43,47 @@ class WorkshopTests(unittest.TestCase):
tool.update_changelog(text, '0.0.109')
class PublishedLookupTests(unittest.TestCase):
def page(self, **overrides):
asset = dict(id='2528466DE220D91C', currentVersionNumber='0.0.147',
gameVersion='1.9.0.37', versions=[dict(version='0.0.147',
assetId='2528466DE220D91C', published=True, approved=True)])
asset.update(overrides)
return '<script id="__NEXT_DATA__" type="application/json">' + json.dumps(
{'props': {'pageProps': {'asset': asset}}}) + '</script>'
def test_public_version_is_not_game_or_bundle_version(self):
self.assertEqual(tool.online_version(self.page(), '2528466DE220D91C'), '0.0.147')
def test_wrong_asset_unpublished_and_ambiguous_pages_are_rejected(self):
cases = [self.page(id='1111111111111111'), self.page(versions=[]),
self.page(private=True), self.page(blocked=True),
self.page(currentVersionNumber='1.9.0.37'),
self.page(versions=[dict(version='0.0.147', assetId='2528466DE220D91C', published=False, approved=True)]),
self.page(versions=[dict(version='0.0.147', assetId='2528466DE220D91C', published=True, approved=False)]),
self.page() + self.page(), '<html>unavailable</html>']
for page in cases:
with self.subTest(page=page), self.assertRaises(ValueError):
tool.online_version(page, '2528466DE220D91C')
def test_stale_response_is_not_a_downgrade(self):
text = tool.update_changelog(TEXT, '0.0.146')
with self.assertRaises(ValueError):
tool.prevent_downgrade(text, '0.0.142')
tool.prevent_downgrade(text, '0.0.146')
tool.prevent_downgrade(text, '0.0.147')
tool.prevent_downgrade(tool.update_changelog(TEXT, '0.0.9'), '0.0.10')
def test_live_sync_validates_response_and_fails_closed(self):
with patch.object(tool.urllib.request, 'urlopen', return_value=io.BytesIO(self.page().encode())):
self.assertEqual(tool.sync_version(PROJECT, TEXT), '0.0.147')
with patch.object(tool.urllib.request, 'urlopen', side_effect=OSError('offline')):
with self.assertRaises(OSError):
tool.sync_version(PROJECT, TEXT)
with patch.object(tool.urllib.request, 'urlopen', return_value=io.BytesIO(self.page().encode())):
with self.assertRaises(ValueError):
tool.sync_version(PROJECT, tool.update_changelog(TEXT, '0.0.148'))
if __name__ == '__main__':
unittest.main()