fix(engine): roll back scheduled turns when flush fails

This commit is contained in:
2026-07-31 14:10:11 +00:00
parent e1a3b3dee0
commit 8ea8e6e26f
2 changed files with 218 additions and 20 deletions
@@ -315,9 +315,7 @@ export class TurnDaemonLifecycle {
await this.commandResponder.publishCommandError(command.requestId, error);
} catch (reportError) {
const reportMessage =
reportError instanceof Error
? reportError.message
: 'Unknown command failure reporting error.';
reportError instanceof Error ? reportError.message : 'Unknown command failure reporting error.';
this.status.lastError = `${this.status.lastError} (failure report: ${reportMessage})`;
}
}
@@ -347,36 +345,31 @@ export class TurnDaemonLifecycle {
const budget = pending.budget ?? this.options.defaultBudget;
const checkpoint = this.status.checkpoint;
let result: TurnRunResult;
let fallbackError = 'Unknown turn daemon error.';
try {
const runProcessor = () => this.processor.run(targetTime, budget, checkpoint);
result = this.stateManager ? await this.stateManager.transaction(runProcessor) : await runProcessor();
const runAndFlush = async (): Promise<TurnRunResult> => {
const nextResult = await this.processor.run(targetTime, budget, checkpoint);
fallbackError = 'Unknown turn flush error.';
this.status.state = 'flushing';
await this.stateStore.saveLastTurnTime(new Date(nextResult.lastTurnTime));
await this.stateStore.saveCheckpoint(nextResult.checkpoint);
await this.hooks?.flushChanges?.(nextResult);
return nextResult;
};
result = this.stateManager ? await this.stateManager.transaction(runAndFlush) : await runAndFlush();
} catch (error) {
this.status.running = false;
this.status.state = 'paused';
this.status.paused = true;
this.errorPaused = true;
this.status.lastError = error instanceof Error ? error.message : 'Unknown turn daemon error.';
this.status.lastError = error instanceof Error ? error.message : fallbackError;
await this.hooks?.onRunError?.(error);
return;
} finally {
this.status.running = false;
}
this.status.state = 'flushing';
try {
await this.stateStore.saveLastTurnTime(new Date(result.lastTurnTime));
await this.stateStore.saveCheckpoint(result.checkpoint);
await this.hooks?.flushChanges?.(result);
} catch (error) {
this.status.state = 'paused';
this.status.paused = true;
this.errorPaused = true;
this.status.lastError = error instanceof Error ? error.message : 'Unknown turn flush error.';
await this.hooks?.onRunError?.(error);
return;
}
await this.applyRunResult(result, startMs);
this.status.state = 'idle';
try {
@@ -75,6 +75,211 @@ describe('TurnDaemonLifecycle', () => {
await loop;
});
it('restores processor and state-store mutations when scheduled flush fails', async () => {
const now = new Date('2026-01-01T00:10:00.000Z');
const previousCheckpoint = {
turnTime: '2026-01-01T00:00:00.000Z',
generalId: 7,
year: 203,
month: 1,
};
const nextCheckpoint = {
turnTime: now.toISOString(),
generalId: 8,
year: 203,
month: 2,
};
let engineState: {
value: string;
lastTurnTime: string;
checkpoint: TurnRunResult['checkpoint'];
} = {
value: 'before',
lastTurnTime: previousCheckpoint.turnTime,
checkpoint: previousCheckpoint,
};
const stateManager = new EngineStateManager();
stateManager.register('test', {
capture: () => structuredClone(engineState),
restore: (snapshot) => {
engineState = snapshot;
},
});
const published = vi.fn();
let resolveError: (() => void) | undefined;
const errorObserved = new Promise<void>((resolve) => {
resolveError = resolve;
});
const lifecycle = new TurnDaemonLifecycle(
{
clock: new ManualClock(now.getTime()),
controlQueue: new InMemoryControlQueue(),
getNextTickTime: () => new Date('2026-01-01T00:05:00.000Z'),
stateStore: {
loadLastTurnTime: async () => new Date(engineState.lastTurnTime),
loadNextGeneralTurnTime: async () => null,
saveLastTurnTime: async (turnTime) => {
engineState.lastTurnTime = turnTime.toISOString();
},
loadCheckpoint: async () => engineState.checkpoint,
saveCheckpoint: async (checkpoint) => {
engineState.checkpoint = checkpoint;
},
},
processor: {
run: async (): Promise<TurnRunResult> => {
engineState.value = 'calculated';
return {
lastTurnTime: now.toISOString(),
processedGenerals: 1,
processedTurns: 1,
durationMs: 0,
partial: false,
checkpoint: nextCheckpoint,
};
},
},
stateManager,
hooks: {
flushChanges: async () => {
expect(engineState).toMatchObject({
value: 'calculated',
lastTurnTime: now.toISOString(),
checkpoint: nextCheckpoint,
});
throw new Error('scheduled flush failed');
},
publishEvents: published,
onRunError: async () => {
resolveError?.();
},
},
},
{
profile: 'test',
defaultBudget: { budgetMs: 100, maxGenerals: 1, catchUpCap: 1 },
}
);
const loop = lifecycle.start();
await errorObserved;
expect(engineState).toEqual({
value: 'before',
lastTurnTime: previousCheckpoint.turnTime,
checkpoint: previousCheckpoint,
});
expect(stateManager.getRevision()).toBe(0);
expect(published).not.toHaveBeenCalled();
expect(lifecycle.getStatus()).toMatchObject({
state: 'paused',
paused: true,
lastError: 'scheduled flush failed',
});
await lifecycle.stop('done');
await loop;
});
it('keeps processor and state-store mutations after flush succeeds and publishes afterward', async () => {
const now = new Date('2026-01-01T00:10:00.000Z');
const previousCheckpoint = {
turnTime: '2026-01-01T00:00:00.000Z',
generalId: 7,
year: 203,
month: 1,
};
const nextCheckpoint = {
turnTime: now.toISOString(),
generalId: 8,
year: 203,
month: 2,
};
let engineState: {
value: string;
lastTurnTime: string;
checkpoint: TurnRunResult['checkpoint'];
} = {
value: 'before',
lastTurnTime: previousCheckpoint.turnTime,
checkpoint: previousCheckpoint,
};
const stateManager = new EngineStateManager();
stateManager.register('test', {
capture: () => structuredClone(engineState),
restore: (snapshot) => {
engineState = snapshot;
},
});
const callOrder: string[] = [];
let resolvePublished: (() => void) | undefined;
const published = new Promise<void>((resolve) => {
resolvePublished = resolve;
});
const lifecycle = new TurnDaemonLifecycle(
{
clock: new ManualClock(now.getTime()),
controlQueue: new InMemoryControlQueue(),
getNextTickTime: (lastTurnTime) => addMinutes(lastTurnTime, 10),
stateStore: {
loadLastTurnTime: async () => new Date(engineState.lastTurnTime),
loadNextGeneralTurnTime: async () => null,
saveLastTurnTime: async (turnTime) => {
callOrder.push('save-last-turn');
engineState.lastTurnTime = turnTime.toISOString();
},
loadCheckpoint: async () => engineState.checkpoint,
saveCheckpoint: async (checkpoint) => {
callOrder.push('save-checkpoint');
engineState.checkpoint = checkpoint;
},
},
processor: {
run: async (): Promise<TurnRunResult> => {
callOrder.push('processor');
engineState.value = 'calculated';
return {
lastTurnTime: now.toISOString(),
processedGenerals: 1,
processedTurns: 1,
durationMs: 0,
partial: false,
checkpoint: nextCheckpoint,
};
},
},
stateManager,
hooks: {
flushChanges: async () => {
callOrder.push('flush');
},
publishEvents: async () => {
callOrder.push('publish');
resolvePublished?.();
},
},
},
{
profile: 'test',
defaultBudget: { budgetMs: 100, maxGenerals: 1, catchUpCap: 1 },
}
);
const loop = lifecycle.start();
await published;
expect(engineState).toEqual({
value: 'calculated',
lastTurnTime: now.toISOString(),
checkpoint: nextCheckpoint,
});
expect(stateManager.getRevision()).toBe(1);
expect(callOrder).toEqual(['processor', 'save-last-turn', 'save-checkpoint', 'flush', 'publish']);
await lifecycle.stop('done');
await loop;
});
it('runs scheduled turn based on queue front and checkpoint context', async () => {
const turnTermMinutes = 10;
const lastTurnTime = new Date(2026, 0, 2, 2, 0, 0, 0);