fix: make Sovereign World heartbeat self-healing

MANIFEST.sha256

@@ -61,6 +61,7 @@ d038c841e9fa2d446f5cd906a01615acb5f613da562a403768fa05c0104b2f2e  tests/browser-
 cece43487e1cd420bba8d44016e6f8cd00cbb05a0ba66ac479cfad42eac95d78  tests/test_sovereign_action_effects.mjs
 0da53afb9207fd45025b4958f8b3299529663cecd3a52e2307b67871e56488c9  tests/test_sovereign_actions.mjs
 fd6d0ef89cc0fffbe625281afdefcff8548b159f6d1d28619b31f4c960887422  tests/test_sovereign_affordance_planner.mjs
+1c22fc56020f0adaa49fcc5ef69edd351f7af2106a6880a8ba8139a2d7139855  tests/test_sovereign_alarm_recovery.mjs
 a48bca09b29b3c6760aeb254ab74ac3d68176dfc4c7b62d122fd2e02f75a6966  tests/test_sovereign_art.mjs
 3d27f2f8f15612fd09408e5eb6f1e8c71b2157586798043420d8732f36facb66  tests/test_sovereign_artifacts.mjs
 3aa402c385f47ed538694ad67edae9a5b635cc3b630e1df98f6215d6fecb962b  tests/test_sovereign_avatar.mjs
@@ -101,13 +102,13 @@ adbb5aa1a3338376a41c12645a5b02a24fcfbbe2808e7b3ab5f056c7e19fd26b  tests/test_sov
 6238a56aa042a20b86e3b555786d34849094c894d8473fd80d6a5255d38a5108  tests/test_sovereign_renderer_motion_upgrade.mjs
 1a5b7048b736543c2582f33e76b5610e423a1378b6a9643ac7bb780b0b0a0b57  tests/test_sovereign_repository_presentation.mjs
 3f63626f40bebc0746521643f1fff6bc522e402218965ee327b8725b6e50dc94  tests/test_sovereign_reproduction.mjs
-f98169e1dcffc3f599a090d4b1f9ab26379f00e13d3b9f7dcb4db9f467134b46  tests/test_sovereign_runtime_contract.mjs
+f936fec7856104ca847a9eabab65c24f0d6dd5cf7c01e1ec81ab31e8f9e8b59c  tests/test_sovereign_runtime_contract.mjs
 6c69813ab0bc4cb1534b82e8a7d06dc323c8a6c0458298e0a0584b7bde4d47e8  tests/test_sovereign_shell.mjs
 3addb20df209d6f6ad84ff37adaa7dd67bf2c10adfc4c372559bc32543311415  tests/test_sovereign_shelter_environment.mjs
 218e8e718765fa3e19aead2359ab5cec36778c6fda596176d8b016a38843da00  tests/test_sovereign_society.mjs
 6bbebe865059f94c75518de6c8d4540e883092805080e5152db8242f95588a25  tests/test_sovereign_strategy.mjs
 5357326e16f8493d08ecc0a56bae1a48773c1f2e979f830f99181054e144ce7a  tests/test_sovereign_transient_physics_upgrade.mjs
-7cdd1568e98eb9ecf4902ff875ba5376e968651e8d6b07ef6ff6d27080fc9dbe  worker/src/index.js
+28c8a7a6c10fe7064172596858f974e4b71b25a3af58419ac23a99a027f022cb  worker/src/index.js
 4b00db1fd2286f8d9dd4b522a9eb1111bd24b4e1befc705b5d6c18b45a4b3c0b  worker/src/neuron-governor.js
 e56357ae087309b86d670288998026568373b7268778157b38612fb38d81eba7  worker/src/persistence.js
 2eac2b46fb9337eb62b4b716be26eb11f92359c58e6c6685039f3506346a09fb  world/actions.js
@@ -122,7 +123,7 @@ a0eb700e098ce6abc2177ce346e5f39de18b2c801bc8af86a99d61f522904adf  world/cognitio
 a1a6c600a884801a45fe446aed7e992dedca6a05a99fac175050c4eb12d8b5fe  world/constants.js
 f1ab5bb183f1fdb33a6e4ad0c1ac53bc5ac07e57a247659ad3002341b721c4a8  world/designs.js
 3bb66b15a8796801310e798ea0e12961ae0744536d7048142e2d452a0b64e18b  world/disease.js
-d6851ab34a7a197acbac1ea0b99831faf2fdb867fee47ab1e0b0c3af849d831e  world/engine.js
+57ed00bce3276e01ba66df8ebf387f9be7437464c731a7fb022133eeac037ec5  world/engine.js
 ac99b325f527a31bf06df73bbd62c65f9076363d2a3fd53211b323157623942e  world/environment.js
 af7f4895306b7bcf4f00e7404d0b1376f970f160f64ec687b605717ae6fa5783  world/epistemics.js
 d7b7cef8b5466204b876f7649f2a80346bfea983dce6a34120aa45243226fefc  world/genesis.js

tests/test_sovereign_alarm_recovery.mjs

@@ -0,0 +1,94 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {SovereignWorld} from '../worker/src/index.js';
+import {createSovereignGenesis,advanceWorldTo,REAL_MS_PER_WORLD_MINUTE} from '../world/index.js';
+
+// Cloudflare owns storage and scheduling. Exercise the real handler with
+// deterministic clock/storage boundaries; do not construct a fake SQL engine.
+function runtime(t){
+  t.mock.method(Date,'now',()=>60_000);
+  const alarms=[];
+  const instance=Object.create(SovereignWorld.prototype);
+  Object.assign(instance,{
+    world:createSovereignGenesis({realEpochMs:59_000}),
+    env:{},lastPersistedWorldMinute:0,
+    ctx:{storage:{getAlarm:async()=>null,setAlarm:async value=>{alarms.push(value);}},getWebSockets:()=>[]},
+  });
+  return {instance,alarms};
+}
+
+test('existing due or future alarms are never postponed during initialization',async t=>{
+  const {instance,alarms}=runtime(t);
+  for(const current of [0,30_000,90_000]){
+    instance.ctx.storage.getAlarm=async()=>current;
+    await instance.ensureAlarm();
+  }
+  assert.deepEqual(alarms,[]);
+});
+
+test('initialization arms a missing alarm one minute ahead',async t=>{
+  const {instance,alarms}=runtime(t);
+  await instance.ensureAlarm();
+  assert.deepEqual(alarms,[120_000]);
+});
+
+test('successful alarm advances the real world and schedules the next heartbeat',async t=>{
+  const {instance,alarms}=runtime(t);
+  instance.world.runtime={lastTickError:'previous failure'};
+  await instance.alarm();
+  assert.ok(instance.world.clock.worldMinute>0);
+  assert.ok(instance.world.citizens.some(c=>c.currentActionId));
+  assert.equal(instance.world.runtime.lastTickError,null);
+  assert.deepEqual(alarms,[120_000]);
+});
+
+test('failed tick is diagnosed and the next alarm can recover',async t=>{
+  const {instance,alarms}=runtime(t);
+  const logs=[];
+  t.mock.method(console,'error',(...args)=>logs.push(args));
+  const original=instance.processCognition;
+  instance.processCognition=async()=>{throw new Error('injected cognition outage');};
+  await assert.doesNotReject(instance.alarm({retryCount:2,isRetry:true}));
+  assert.match(instance.world.runtime.lastTickError,/injected cognition outage/);
+  assert.equal(instance.world.runtime.lastAlarmRetryCount,2);
+  assert.equal(logs[0][0],'CYMONIA_TICK_FAILED');
+  assert.deepEqual(alarms,[120_000]);
+  instance.processCognition=original;
+  await instance.alarm();
+  assert.equal(instance.world.runtime.lastTickError,null);
+  assert.deepEqual(alarms,[120_000,120_000]);
+});
+
+test('rescheduling failure remains visible to Cloudflare for retry',async t=>{
+  const {instance}=runtime(t);
+  instance.ctx.storage.setAlarm=async()=>{throw new Error('storage unavailable');};
+  await assert.rejects(instance.alarm(),/storage unavailable/);
+});
+
+test('one malformed local goal falls back to observation while peers continue',()=>{
+  const world=createSovereignGenesis({realEpochMs:0});
+  const citizen=world.citizens[0];
+  citizen.activeGoal={actionTypes:{invalid:true}};
+  assert.doesNotThrow(()=>advanceWorldTo(world,REAL_MS_PER_WORLD_MINUTE));
+  assert.equal(world.clock.worldMinute,1);
+  assert.equal(world.actions.find(a=>a.id===citizen.currentActionId)?.type,'OBSERVE');
+  assert.ok(world.citizens.slice(1).some(c=>c.currentActionId));
+  assert.ok(world.ledger.some(e=>e.type==='LOCAL_COGNITION_DEFERRED'&&e.actorId===citizen.id));
+});
+
+test('fallback action failure is propagated instead of silently reported as recovery',()=>{
+  const world=createSovereignGenesis({realEpochMs:0});
+  Object.freeze(world.actions);
+  assert.throws(()=>advanceWorldTo(world,REAL_MS_PER_WORLD_MINUTE),TypeError);
+  assert.ok(world.ledger.some(e=>e.type==='LOCAL_PLAN_REJECTED'));
+});
+
+test('rejected local plan starts observation and does not block peer actions',()=>{
+  const world=createSovereignGenesis({realEpochMs:0});
+  const citizen=world.citizens[0];
+  citizen.plans=null;
+  assert.doesNotThrow(()=>advanceWorldTo(world,REAL_MS_PER_WORLD_MINUTE));
+  assert.equal(world.actions.find(a=>a.id===citizen.currentActionId)?.type,'OBSERVE');
+  assert.ok(world.citizens.slice(1).some(c=>c.currentActionId));
+  assert.ok(world.ledger.some(e=>e.type==='LOCAL_PLAN_REJECTED'&&e.actorId===citizen.id));
+});

tests/test_sovereign_runtime_contract.mjs

@@ -23,36 +23,10 @@ test('hibernating websocket runtime survives object eviction without volatile cl
   assert.doesNotMatch(worker,/this\.clients/);
 });
 
-test('constructor never postpones a due alarm and alarm handler keeps the schedule alive',()=>{
-  assert.match(worker,/const ALARM_MS=60_000/);
-
-  const ensureStart=worker.indexOf('async ensureAlarm(){');
-  const ensureEnd=worker.indexOf('persist(options={})',ensureStart);
-  const ensure=worker.slice(ensureStart,ensureEnd);
-
-  assert.match(ensure,/getAlarm\(\)/);
-  assert.match(ensure,/current===null/);
-  assert.match(ensure,/setAlarm\(Date\.now\(\)\+ALARM_MS\)/);
-  assert.doesNotMatch(ensure,/current\s*<\s*Date\.now\(\)/);
-
-  const tickStart=worker.indexOf('async tick(){');
-  const tickEnd=worker.indexOf('async alarm(){',tickStart);
-  const tick=worker.slice(tickStart,tickEnd);
-
-  assert.doesNotMatch(tick,/ensureAlarm/);
-  assert.doesNotMatch(tick,/setAlarm/);
-
-  const alarmStart=worker.indexOf('async alarm(){');
-  const alarmEnd=worker.indexOf('async processCognition',alarmStart);
-  const alarm=worker.slice(alarmStart,alarmEnd);
-
-  assert.match(alarm,/await this\.tick\(\)/);
-  assert.match(alarm,/finally/);
-  assert.match(alarm,/setAlarm\(Date\.now\(\)\+ALARM_MS\)/);
-});
 test('runtime recovery is persisted immediately before hibernation can discard the rebase',()=>{
   const tickStart=worker.indexOf('async tick(){');
-  const tickEnd=worker.indexOf('async alarm(){',tickStart);
+  const tickEnd=worker.indexOf('async alarm(',tickStart);
+  assert.ok(tickStart>=0&&tickEnd>tickStart,'tick method boundaries must exist');
   const tick=worker.slice(tickStart,tickEnd);
 
   assert.match(tick,/progress\.recovered/);

worker/src/index.js

@@ -305,11 +305,39 @@ export class SovereignWorld {
       await this.persist();
     }
   }
-  async alarm(){
+  async alarm(alarmInfo){
+    const startedAt=Date.now();
+    const runtime=ensureRuntime(this.world);
+
     try{
       await this.tick();
+
+      runtime.lastTickRealMs=Date.now();
+      runtime.lastTickWorldMinute=this.world.clock.worldMinute;
+      runtime.lastTickError=null;
+      runtime.lastAlarmRetryCount=Number(alarmInfo?.retryCount||0);
+    }catch(error){
+      runtime.lastTickRealMs=Date.now();
+      runtime.lastTickWorldMinute=this.world?.clock?.worldMinute??null;
+      runtime.lastTickError=String(error?.stack||error?.message||error).slice(0,1000);
+      runtime.lastAlarmRetryCount=Number(alarmInfo?.retryCount||0);
+
+      console.error('CYMONIA_TICK_FAILED',JSON.stringify({
+        worldMinute:this.world?.clock?.worldMinute??null,
+        retryCount:Number(alarmInfo?.retryCount||0),
+        isRetry:Boolean(alarmInfo?.isRetry),
+        elapsedMs:Date.now()-startedAt,
+        error:String(error?.message||error).slice(0,300)
+      }));
+
+      // Do not rethrow here.
+      // A single malformed Citizen or transient runtime error must never
+      // permanently stop the Sovereign World heartbeat.
     }finally{
-      await this.ctx.storage.setAlarm(Date.now()+ALARM_MS);
+      const nextAlarm=Date.now()+ALARM_MS;
+      runtime.nextAlarmRealMs=nextAlarm;
+
+      await this.ctx.storage.setAlarm(nextAlarm);
     }
   }
   async processCognition(limit){