fix: recover stale in-memory reserved turn leases
This commit is contained in:
@@ -241,12 +241,20 @@ export class InMemoryReservedTurnStore {
|
|||||||
return new Date(Date.now() + this.leaseDurationMs);
|
return new Date(Date.now() + this.leaseDurationMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async acquireGeneralLease(generalId: number): Promise<void> {
|
private async acquireGeneralLease(generalId: number): Promise<boolean> {
|
||||||
const revisionStore = this.prisma.generalTurnRevision;
|
const revisionStore = this.prisma.generalTurnRevision;
|
||||||
if (!revisionStore) {
|
if (!revisionStore) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
|
const previous = (await revisionStore.findUnique({ where: { generalId } })) as {
|
||||||
|
leaseOwner: string | null;
|
||||||
|
leaseExpiresAt: Date | null;
|
||||||
|
} | null;
|
||||||
|
const retainedExistingLease =
|
||||||
|
previous?.leaseOwner === this.leaseOwner &&
|
||||||
|
previous.leaseExpiresAt !== null &&
|
||||||
|
previous.leaseExpiresAt.getTime() > now.getTime();
|
||||||
const leaseExpiresAt = this.getLeaseExpiresAt();
|
const leaseExpiresAt = this.getLeaseExpiresAt();
|
||||||
let claimed = await revisionStore.updateMany({
|
let claimed = await revisionStore.updateMany({
|
||||||
where: {
|
where: {
|
||||||
@@ -280,14 +288,22 @@ export class InMemoryReservedTurnStore {
|
|||||||
throw new ReservedTurnLeaseConflictError(`general:${generalId}`);
|
throw new ReservedTurnLeaseConflictError(`general:${generalId}`);
|
||||||
}
|
}
|
||||||
this.leasedGeneralIds.add(generalId);
|
this.leasedGeneralIds.add(generalId);
|
||||||
|
return !retainedExistingLease;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async acquireNationLease(nationId: number, officerLevel: number): Promise<void> {
|
private async acquireNationLease(nationId: number, officerLevel: number): Promise<boolean> {
|
||||||
const revisionStore = this.prisma.nationTurnRevision;
|
const revisionStore = this.prisma.nationTurnRevision;
|
||||||
if (!revisionStore) {
|
if (!revisionStore) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
|
const previous = (await revisionStore.findUnique({
|
||||||
|
where: { nationId_officerLevel: { nationId, officerLevel } },
|
||||||
|
})) as { leaseOwner: string | null; leaseExpiresAt: Date | null } | null;
|
||||||
|
const retainedExistingLease =
|
||||||
|
previous?.leaseOwner === this.leaseOwner &&
|
||||||
|
previous.leaseExpiresAt !== null &&
|
||||||
|
previous.leaseExpiresAt.getTime() > now.getTime();
|
||||||
const leaseExpiresAt = this.getLeaseExpiresAt();
|
const leaseExpiresAt = this.getLeaseExpiresAt();
|
||||||
let claimed = await revisionStore.updateMany({
|
let claimed = await revisionStore.updateMany({
|
||||||
where: {
|
where: {
|
||||||
@@ -323,6 +339,7 @@ export class InMemoryReservedTurnStore {
|
|||||||
throw new ReservedTurnLeaseConflictError(`nation:${nationId}:${officerLevel}`);
|
throw new ReservedTurnLeaseConflictError(`nation:${nationId}:${officerLevel}`);
|
||||||
}
|
}
|
||||||
this.leasedNationKeys.add(buildNationKey(nationId, officerLevel));
|
this.leasedNationKeys.add(buildNationKey(nationId, officerLevel));
|
||||||
|
return !retainedExistingLease;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async releaseGeneralLease(generalId: number): Promise<void> {
|
private async releaseGeneralLease(generalId: number): Promise<void> {
|
||||||
@@ -348,23 +365,32 @@ export class InMemoryReservedTurnStore {
|
|||||||
const hadGeneralLease = this.leasedGeneralIds.has(generalId);
|
const hadGeneralLease = this.leasedGeneralIds.has(generalId);
|
||||||
const nationKey = nation ? buildNationKey(nation.nationId, nation.officerLevel) : null;
|
const nationKey = nation ? buildNationKey(nation.nationId, nation.officerLevel) : null;
|
||||||
const hadNationLease = nationKey ? this.leasedNationKeys.has(nationKey) : false;
|
const hadNationLease = nationKey ? this.leasedNationKeys.has(nationKey) : false;
|
||||||
|
let acquiredFreshGeneralLease = false;
|
||||||
|
let acquiredFreshNationLease = false;
|
||||||
try {
|
try {
|
||||||
await this.acquireGeneralLease(generalId);
|
acquiredFreshGeneralLease = await this.acquireGeneralLease(generalId);
|
||||||
if (nation) {
|
if (nation) {
|
||||||
await this.acquireNationLease(nation.nationId, nation.officerLevel);
|
acquiredFreshNationLease = await this.acquireNationLease(nation.nationId, nation.officerLevel);
|
||||||
}
|
}
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
// A newly acquired lease starts a fresh API/daemon ownership boundary.
|
// A newly acquired lease starts a fresh API/daemon ownership boundary.
|
||||||
// Re-read PostgreSQL even if a prior run left a stale dirty marker;
|
// Re-read PostgreSQL even if a prior run left a stale dirty marker;
|
||||||
// repeated access under the same held lease keeps local mutations.
|
// repeated access under the same held lease keeps local mutations.
|
||||||
this.refreshGeneralTurns(generalId, !hadGeneralLease),
|
this.refreshGeneralTurns(generalId, acquiredFreshGeneralLease),
|
||||||
nation ? this.refreshNationTurns(nation.nationId, nation.officerLevel) : Promise.resolve(),
|
nation
|
||||||
|
? this.refreshNationTurns(nation.nationId, nation.officerLevel, acquiredFreshNationLease)
|
||||||
|
: Promise.resolve(),
|
||||||
]);
|
]);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (nation && nationKey !== null && !hadNationLease && this.leasedNationKeys.has(nationKey)) {
|
if (
|
||||||
|
nation &&
|
||||||
|
nationKey !== null &&
|
||||||
|
(!hadNationLease || acquiredFreshNationLease) &&
|
||||||
|
this.leasedNationKeys.has(nationKey)
|
||||||
|
) {
|
||||||
await this.releaseNationLease(nation.nationId, nation.officerLevel);
|
await this.releaseNationLease(nation.nationId, nation.officerLevel);
|
||||||
}
|
}
|
||||||
if (!hadGeneralLease && this.leasedGeneralIds.has(generalId)) {
|
if ((!hadGeneralLease || acquiredFreshGeneralLease) && this.leasedGeneralIds.has(generalId)) {
|
||||||
await this.releaseGeneralLease(generalId);
|
await this.releaseGeneralLease(generalId);
|
||||||
}
|
}
|
||||||
throw error;
|
throw error;
|
||||||
@@ -406,9 +432,9 @@ export class InMemoryReservedTurnStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async refreshNationTurns(nationId: number, officerLevel: number): Promise<void> {
|
async refreshNationTurns(nationId: number, officerLevel: number, force = false): Promise<void> {
|
||||||
const key = buildNationKey(nationId, officerLevel);
|
const key = buildNationKey(nationId, officerLevel);
|
||||||
if (this.dirtyNationKeys.has(key) || this.pendingNationInitializationKeys.has(key)) {
|
if (!force && (this.dirtyNationKeys.has(key) || this.pendingNationInitializationKeys.has(key))) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const rows = await this.prisma.nationTurn.findMany({
|
const rows = await this.prisma.nationTurn.findMany({
|
||||||
|
|||||||
@@ -1,10 +1,7 @@
|
|||||||
import { describe, expect, it, vi } from 'vitest';
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
import { asRecord } from '@sammo-ts/common';
|
import { asRecord } from '@sammo-ts/common';
|
||||||
|
|
||||||
import {
|
import { InMemoryReservedTurnStore, ReservedTurnLeaseConflictError } from '../src/turn/reservedTurnStore.js';
|
||||||
InMemoryReservedTurnStore,
|
|
||||||
ReservedTurnLeaseConflictError,
|
|
||||||
} from '../src/turn/reservedTurnStore.js';
|
|
||||||
|
|
||||||
interface RevisionRow {
|
interface RevisionRow {
|
||||||
revision: number;
|
revision: number;
|
||||||
@@ -202,17 +199,20 @@ describe('reserved turn daemon lease', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('refreshes a stale dirty cache after acquiring a fresh lease but preserves mutations under the held lease', async () => {
|
it('refreshes after a durable flush whose in-memory lease acknowledgement was interrupted', async () => {
|
||||||
const harness = buildHarness();
|
const harness = buildHarness();
|
||||||
harness.store.setGeneralTurn(7, 0, { action: '휴식', args: {} });
|
await harness.store.prepareTurnsForExecution(7);
|
||||||
|
harness.store.shiftGeneralTurns(7, -1);
|
||||||
|
const changes = harness.store.peekDirtyState();
|
||||||
|
await harness.store.persistChanges(harness.prisma, changes);
|
||||||
|
|
||||||
|
expect(harness.store.getGeneralTurn(7, 0).action).toBe('휴식');
|
||||||
|
expect(harness.store.peekDirtyState()).toMatchObject({ generalIds: [7], generalLeaseIds: [7] });
|
||||||
|
expect(harness.getRevision()).toMatchObject({ revision: 1, leaseOwner: null });
|
||||||
|
|
||||||
await harness.store.prepareTurnsForExecution(7);
|
await harness.store.prepareTurnsForExecution(7);
|
||||||
expect(harness.store.getGeneralTurn(7, 0).action).toBe('che_훈련');
|
expect(harness.store.getGeneralTurn(7, 0).action).toBe('che_훈련');
|
||||||
|
expect(harness.generalFindMany).toHaveBeenCalledTimes(2);
|
||||||
harness.store.setGeneralTurn(7, 0, { action: 'che_사기진작', args: {} });
|
|
||||||
await harness.store.prepareTurnsForExecution(7);
|
|
||||||
expect(harness.store.getGeneralTurn(7, 0).action).toBe('che_사기진작');
|
|
||||||
expect(harness.generalFindMany).toHaveBeenCalledOnce();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('rejects an active foreign lease before reading the queue', async () => {
|
it('rejects an active foreign lease before reading the queue', async () => {
|
||||||
@@ -222,9 +222,7 @@ describe('reserved turn daemon lease', () => {
|
|||||||
leaseExpiresAt: new Date(Date.now() + 60_000),
|
leaseExpiresAt: new Date(Date.now() + 60_000),
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(harness.store.prepareTurnsForExecution(7)).rejects.toBeInstanceOf(
|
await expect(harness.store.prepareTurnsForExecution(7)).rejects.toBeInstanceOf(ReservedTurnLeaseConflictError);
|
||||||
ReservedTurnLeaseConflictError
|
|
||||||
);
|
|
||||||
expect(harness.generalFindMany).not.toHaveBeenCalled();
|
expect(harness.generalFindMany).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user