From 693906c6a89f36e2cc7e3b5a276f3f2f49a17785 Mon Sep 17 00:00:00 2001 From: hided62 Date: Mon, 27 Jul 2026 00:20:46 +0000 Subject: [PATCH 1/6] test live sortie nation conflict cleanup --- .../test/liveSortiePersistence.integration.test.ts | 1 + .../turnCommandCoreReference.integration.test.ts | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/tools/integration-tests/test/liveSortiePersistence.integration.test.ts b/tools/integration-tests/test/liveSortiePersistence.integration.test.ts index 4494bd7..cc5e2eb 100644 --- a/tools/integration-tests/test/liveSortiePersistence.integration.test.ts +++ b/tools/integration-tests/test/liveSortiePersistence.integration.test.ts @@ -119,6 +119,7 @@ integration('live sortie PostgreSQL persistence retry', () => { it.each([ ['conquest and nation collapse', 'live-sortie-conquest.json'], + ['collapsed nation conflict cleanup', 'live-sortie-collapse-conflict.json'], ['single defending general', 'live-sortie-defender.json'], ['multiple defending generals', 'live-sortie-multiple-defenders.json'], ['supply retreat', 'live-sortie-supply-retreat.json'], diff --git a/tools/integration-tests/test/turnCommandCoreReference.integration.test.ts b/tools/integration-tests/test/turnCommandCoreReference.integration.test.ts index fcf0b74..b332d0d 100644 --- a/tools/integration-tests/test/turnCommandCoreReference.integration.test.ts +++ b/tools/integration-tests/test/turnCommandCoreReference.integration.test.ts @@ -93,6 +93,10 @@ integration('core ↔ legacy command-boundary differential', () => { it.each([ ['nation declaration', 'fixtures/turn-differential/nation-declaration.json'], ['live sortie conquest', 'fixtures/turn-differential/live-sortie-conquest.json'], + [ + 'live sortie collapsed nation conflict cleanup', + 'fixtures/turn-differential/live-sortie-collapse-conflict.json', + ], ['live sortie against a defending general', 'fixtures/turn-differential/live-sortie-defender.json'], [ 'live sortie against multiple defending generals', @@ -104,6 +108,10 @@ integration('core ↔ legacy command-boundary differential', () => { ['live sortie conflict arbitration', 'fixtures/turn-differential/live-sortie-conflict-arbitration.json'], ['live sortie tied conflict', 'fixtures/turn-differential/live-sortie-conflict-tie.json'], ['live sortie outer lifecycle: conquest', 'fixtures/turn-differential/live-sortie-conquest.json'], + [ + 'live sortie outer lifecycle: collapsed nation conflict cleanup', + 'fixtures/turn-differential/live-sortie-collapse-conflict.json', + ], ['live sortie outer lifecycle: defender', 'fixtures/turn-differential/live-sortie-defender.json'], [ 'live sortie outer lifecycle: multiple defenders', @@ -173,6 +181,12 @@ integration('core ↔ legacy command-boundary differential', () => { }); expect(reference.after.cities.find((city) => city.id === 71)?.supplyState).toBe(1); } + if (fixturePath.endsWith('live-sortie-collapse-conflict.json')) { + expect(reference.before.cities.find((city) => city.id === 71)?.conflict).toEqual({ 2: 1234 }); + expect(reference.after.nations.some((nation) => nation.id === 2)).toBe(false); + expect(Object.keys(reference.after.cities.find((city) => city.id === 71)?.conflict ?? {})).toEqual([]); + expect(Object.keys(core.after.cities.find((city) => city.id === 71)?.conflict ?? {})).toEqual([]); + } if (fixturePath.endsWith('live-sortie-conflict-arbitration.json')) { expect(reference.before.cities.find((city) => city.id === 70)?.conflict).toEqual({ 3: 10_000 }); expect(reference.after.cities.find((city) => city.id === 70)).toMatchObject({ From 87448dff9ce79f29c051254a70c1895818108a74 Mon Sep 17 00:00:00 2001 From: hided62 Date: Mon, 27 Jul 2026 00:34:55 +0000 Subject: [PATCH 2/6] test monthly disaster database parity --- ...yDisasterCoreReference.integration.test.ts | 296 ++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 tools/integration-tests/test/monthlyDisasterCoreReference.integration.test.ts diff --git a/tools/integration-tests/test/monthlyDisasterCoreReference.integration.test.ts b/tools/integration-tests/test/monthlyDisasterCoreReference.integration.test.ts new file mode 100644 index 0000000..d426eb8 --- /dev/null +++ b/tools/integration-tests/test/monthlyDisasterCoreReference.integration.test.ts @@ -0,0 +1,296 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { describe, expect, it } from 'vitest'; +import { loadActionModuleBundle, type City, type GeneralActionModule, type MapDefinition } from '@sammo-ts/logic'; +import { InMemoryTurnWorld } from '@sammo-ts/game-engine/turn/inMemoryWorld.js'; +import { createRaiseDisasterHandler } from '@sammo-ts/game-engine/turn/monthlyDisasterAction.js'; +import type { TurnEvent, TurnGeneral, TurnWorldSnapshot, TurnWorldState } from '@sammo-ts/game-engine/turn/types.js'; + +import { findTurnDifferentialWorkspaceRoot } from '../src/turn-differential/referenceSnapshot.js'; + +type ReferenceGeneral = { + id: number; + injury: number; + crew: number; + train: number; + atmos: number; + item: string; +}; + +type ReferenceCity = { + id: number; + state: number; + population: number; + agriculture: number; + commerce: number; + security: number; + trust: number; + defence: number; + wall: number; +}; + +type ReferenceLog = { + generalId?: number; + text: string; + scope?: string; + category?: string; +}; + +type ReferenceMonthlyTrace = { + action: string; + before: { + watermarks: { + logId: number; + }; + }; + after: { + logs: ReferenceLog[]; + }; + beforeDetails: { + generals: ReferenceGeneral[]; + cities: ReferenceCity[]; + worldHistory: ReferenceLog[]; + }; + afterDetails: { + generals: ReferenceGeneral[]; + cities: ReferenceCity[]; + worldHistory: ReferenceLog[]; + }; +}; + +const configuredWorkspaceRoot = process.env.TURN_DIFFERENTIAL_WORKSPACE_ROOT; +const workspaceRoot = configuredWorkspaceRoot ?? findTurnDifferentialWorkspaceRoot(process.cwd()); +const integration = describe.skipIf(!workspaceRoot || process.env.TURN_DIFFERENTIAL_REFERENCE !== '1'); + +const fixtureRelativePath = 'fixtures/monthly-differential/raise-disaster-january.json'; + +const runReferenceMonthlyTrace = (): ReferenceMonthlyTrace => { + const stackDirectory = path.join(workspaceRoot!, 'docker_compose_files/reference'); + const fixture = fs.readFileSync(path.join(stackDirectory, fixtureRelativePath), 'utf8'); + const runnerScript = + process.env.MONTHLY_DIFFERENTIAL_RUNNER_SCRIPT ?? + path.join(workspaceRoot!, 'ref/sam/hwe/compare/monthly_event_trace.php'); + const stdout = execFileSync('./scripts/run-turn-differential-case.sh', ['-'], { + cwd: stackDirectory, + input: fixture, + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + env: { + ...process.env, + TURN_DIFFERENTIAL_RUNNER_SCRIPT: runnerScript, + }, + }); + return JSON.parse(stdout) as ReferenceMonthlyTrace; +}; + +const map: MapDefinition = { + id: 'monthly-disaster-reference', + name: 'monthly-disaster-reference', + cities: [], + defaults: { trust: 50, trade: 100, supplyState: 1, frontState: 0 }, +}; + +const city: City = { + id: 1, + name: '재난도시', + nationId: 0, + level: 1, + state: 5, + population: 1_001, + populationMax: 2_000, + agriculture: 501, + agricultureMax: 1_000, + commerce: 499, + commerceMax: 1_000, + security: 0, + securityMax: 100, + supplyState: 1, + frontState: 2, + defence: 99, + defenceMax: 1_000, + wall: 101, + wallMax: 1_000, + conflict: {}, + meta: { trust: 99.5, trade: 100 }, +}; + +const buildGeneral = (id: number, name: string, injury: number, item: string | null): TurnGeneral => ({ + id, + name, + nationId: 0, + cityId: 1, + troopId: 0, + stats: { leadership: 50, strength: 50, intelligence: 50 }, + experience: 0, + dedication: 0, + officerLevel: 1, + role: { + personality: null, + specialDomestic: null, + specialWar: null, + items: { horse: null, weapon: null, book: null, item }, + }, + injury, + gold: 1_000, + rice: 1_000, + crew: 99, + crewTypeId: 1_100, + train: 51, + atmos: 50, + age: 20, + npcState: 0, + triggerState: { flags: {}, counters: {}, modifiers: {}, meta: {} }, + meta: { killturn: 24 }, + turnTime: new Date('0193-01-01T00:00:00.000Z'), +}); + +const projectReferenceGenerals = (generals: ReferenceGeneral[]) => + generals.map((general) => ({ + id: general.id, + injury: general.injury, + crew: general.crew, + train: general.train, + atmos: general.atmos, + item: general.item === 'None' ? null : general.item, + })); + +const projectCoreGenerals = (world: InMemoryTurnWorld) => + world + .listGenerals() + .sort((left, right) => left.id - right.id) + .map((general) => ({ + id: general.id, + injury: general.injury, + crew: general.crew, + train: general.train, + atmos: general.atmos, + item: general.role.items.item, + })); + +const projectReferenceCity = (citySnapshot: ReferenceCity) => ({ + state: citySnapshot.state, + population: citySnapshot.population, + agriculture: citySnapshot.agriculture, + commerce: citySnapshot.commerce, + security: citySnapshot.security, + trust: citySnapshot.trust, + defence: citySnapshot.defence, + wall: citySnapshot.wall, +}); + +const projectCoreCity = (citySnapshot: City) => ({ + state: citySnapshot.state, + population: citySnapshot.population, + agriculture: citySnapshot.agriculture, + commerce: citySnapshot.commerce, + security: citySnapshot.security, + trust: citySnapshot.meta.trust, + defence: citySnapshot.defence, + wall: citySnapshot.wall, +}); + +const normalizeStoredLogText = (value: string): string => + value.replace(/^(?:●<\/>|◆<\/>|★<\/>)(?:(?:\d+년 )?\d+월:|\d+년:)?/, ''); + +const projectReferenceLogs = (trace: ReferenceMonthlyTrace) => [ + ...trace.afterDetails.worldHistory.map((entry) => ({ + scope: 'system', + category: 'history', + generalId: null, + text: normalizeStoredLogText(entry.text), + })), + ...trace.after.logs + .filter((entry) => entry.generalId !== undefined) + .map((entry) => ({ + scope: entry.scope, + category: entry.category, + generalId: entry.generalId ?? null, + text: normalizeStoredLogText(entry.text), + })), +]; + +const buildCoreWorld = async (generalActionModules: GeneralActionModule[]): Promise => { + const state: TurnWorldState = { + id: 1, + currentYear: 193, + currentMonth: 1, + tickSeconds: 600, + lastTurnTime: new Date('0193-01-01T00:00:00.000Z'), + meta: { hiddenSeed: 'disaster-test-6' }, + }; + const snapshot: TurnWorldSnapshot = { + scenarioConfig: { + stat: { total: 300, min: 10, max: 100, npcTotal: 150, npcMax: 50, npcMin: 10, chiefMin: 70 }, + iconPath: '', + map: {}, + const: {}, + environment: { mapName: map.id, unitSet: 'default' }, + }, + scenarioMeta: { + title: 'monthly disaster reference', + startYear: 190, + life: null, + fiction: null, + history: [], + ignoreDefaultEvents: false, + }, + map, + diplomacy: [], + events: [], + initialEvents: [], + generals: [ + buildGeneral(1, '보호장수', 0, 'che_부적_태현청생부'), + buildGeneral(3, '부상장수', 0, null), + buildGeneral(5, '중상장수', 70, null), + ], + cities: [city], + nations: [], + troops: [], + }; + const world = new InMemoryTurnWorld(state, snapshot, { + schedule: { entries: [{ startMinute: 0, tickMinutes: 10 }] }, + }); + const event: TurnEvent = { + id: 1, + targetCode: 'month', + priority: 1_000, + condition: true, + action: [['RaiseDisaster']], + meta: {}, + }; + await createRaiseDisasterHandler({ + getWorld: () => world, + generalActionModules, + })([], { year: 193, month: 1, startyear: 190, currentEventID: event.id, turnTime: state.lastTurnTime }, event); + return world; +}; + +integration('RaiseDisaster core ↔ legacy database snapshot', () => { + it('matches city damage, item-aware injury, and persisted logs', async () => { + const reference = runReferenceMonthlyTrace(); + expect(reference.action).toBe('RaiseDisaster'); + expect(projectReferenceGenerals(reference.beforeDetails.generals)).toEqual([ + { id: 1, injury: 0, crew: 99, train: 51, atmos: 50, item: 'che_부적_태현청생부' }, + { id: 3, injury: 0, crew: 99, train: 51, atmos: 50, item: null }, + { id: 5, injury: 70, crew: 99, train: 51, atmos: 50, item: null }, + ]); + + const moduleBundle = await loadActionModuleBundle(); + const core = await buildCoreWorld(moduleBundle.general); + const coreCity = projectCoreCity(core.getCityById(1)!); + const referenceCity = projectReferenceCity(reference.afterDetails.cities[0]!); + expect({ ...coreCity, trust: undefined }).toEqual({ ...referenceCity, trust: undefined }); + expect(coreCity.trust).toBeCloseTo(referenceCity.trust); + expect(projectCoreGenerals(core)).toEqual(projectReferenceGenerals(reference.afterDetails.generals)); + + const coreLogs = core.peekDirtyState().logs.map((entry) => ({ + scope: String(entry.scope).toLowerCase(), + category: String(entry.category).toLowerCase(), + generalId: entry.generalId ?? null, + text: entry.text, + })); + expect(coreLogs).toEqual(projectReferenceLogs(reference)); + }, 120_000); +}); From bff7b05c052f52be7e1416e54ebb18ac29c62de9 Mon Sep 17 00:00:00 2001 From: hided62 Date: Mon, 27 Jul 2026 00:46:22 +0000 Subject: [PATCH 3/6] test turn daemon process kill takeover --- .../test/helpers/turnDaemonLeaseHolder.mjs | 31 ++++++ .../test/turnDaemonLease.integration.test.ts | 96 +++++++++++++++++++ tools/run-conditional-integration.sh | 1 + 3 files changed, 128 insertions(+) create mode 100644 app/game-engine/test/helpers/turnDaemonLeaseHolder.mjs diff --git a/app/game-engine/test/helpers/turnDaemonLeaseHolder.mjs b/app/game-engine/test/helpers/turnDaemonLeaseHolder.mjs new file mode 100644 index 0000000..c4b253b --- /dev/null +++ b/app/game-engine/test/helpers/turnDaemonLeaseHolder.mjs @@ -0,0 +1,31 @@ +import { DatabaseTurnDaemonLease } from '../../src/lifecycle/databaseTurnDaemonLease.ts'; + +const databaseUrl = process.env.TURN_DAEMON_LEASE_DATABASE_URL; +const profile = process.env.TURN_DAEMON_LEASE_PROFILE; +const ownerId = process.env.TURN_DAEMON_LEASE_OWNER_ID; +const leaseDurationMs = Number(process.env.TURN_DAEMON_LEASE_DURATION_MS); + +if (!databaseUrl || !profile || !ownerId || !Number.isInteger(leaseDurationMs)) { + throw new Error('lease holder requires database URL, profile, owner, and duration'); +} + +const lease = await DatabaseTurnDaemonLease.connect(databaseUrl, { + profile, + ownerId, + leaseDurationMs, + heartbeat: true, +}); +const token = await lease.acquire(); +if (!token) { + throw new Error('lease holder could not acquire the profile lease'); +} + +process.stdout.write( + `${JSON.stringify({ + profile: token.profile, + ownerId: token.ownerId, + fencingEpoch: token.fencingEpoch.toString(), + })}\n` +); + +setInterval(() => undefined, 60_000); diff --git a/app/game-engine/test/turnDaemonLease.integration.test.ts b/app/game-engine/test/turnDaemonLease.integration.test.ts index e5c5ba7..b4d7000 100644 --- a/app/game-engine/test/turnDaemonLease.integration.test.ts +++ b/app/game-engine/test/turnDaemonLease.integration.test.ts @@ -1,3 +1,7 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { fileURLToPath } from 'node:url'; + import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; @@ -7,6 +11,41 @@ import { DatabaseTurnDaemonLease, TurnDaemonLeaseLostError } from '../src/lifecy const databaseUrl = process.env.TURN_DAEMON_LEASE_DATABASE_URL ?? process.env.INPUT_EVENT_DATABASE_URL; const integration = describe.skipIf(!databaseUrl); const profilePrefix = 'integration:turn-lease:'; +const holderScript = fileURLToPath(new URL('./helpers/turnDaemonLeaseHolder.mjs', import.meta.url)); + +type HolderReady = { + profile: string; + ownerId: string; + fencingEpoch: string; +}; + +const waitForHolderReady = async (child: ReturnType, timeoutMs = 10_000): Promise => { + let output = ''; + const ready = new Promise((resolve, reject) => { + child.stdout?.setEncoding('utf8'); + child.stdout?.on('data', (chunk: string) => { + output += chunk; + const newline = output.indexOf('\n'); + if (newline < 0) { + return; + } + resolve(JSON.parse(output.slice(0, newline)) as HolderReady); + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + reject(new Error(`lease holder exited before readiness: code=${String(code)} signal=${String(signal)}`)); + }); + }); + const timeout = new Promise((_, reject) => { + setTimeout(() => reject(new Error('lease holder readiness timed out')), timeoutMs).unref(); + }); + return Promise.race([ready, timeout]); +}; + +const delay = (durationMs: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, durationMs); + }); integration('database turn daemon lease and fencing', () => { let db: GamePrismaClient; @@ -115,4 +154,61 @@ integration('database turn daemon lease and fencing', () => { expect((await second.acquire())?.fencingEpoch).toBe(2n); await expect(first.assertActive()).rejects.toBeInstanceOf(TurnDaemonLeaseLostError); }); + + it('takes over after a heartbeat owner process is killed and its lease expires', async () => { + const profile = `${profilePrefix}process-kill`; + const leaseDurationMs = 1_200; + const holder = spawn(process.execPath, ['--experimental-strip-types', holderScript], { + env: { + ...process.env, + TURN_DAEMON_LEASE_DATABASE_URL: databaseUrl!, + TURN_DAEMON_LEASE_PROFILE: profile, + TURN_DAEMON_LEASE_OWNER_ID: 'process-owner', + TURN_DAEMON_LEASE_DURATION_MS: String(leaseDurationMs), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let successor: DatabaseTurnDaemonLease | null = null; + try { + await expect(waitForHolderReady(holder)).resolves.toEqual({ + profile, + ownerId: 'process-owner', + fencingEpoch: '1', + }); + expect(await db.turnDaemonLease.findUniqueOrThrow({ where: { profile } })).toMatchObject({ + ownerId: 'process-owner', + fencingEpoch: 1n, + }); + + expect(holder.kill('SIGKILL')).toBe(true); + const [exitCode, signal] = (await once(holder, 'exit')) as [number | null, NodeJS.Signals | null]; + expect(exitCode).toBeNull(); + expect(signal).toBe('SIGKILL'); + + successor = await createLease(profile, 'successor-owner'); + expect(await successor.acquire()).toBeNull(); + + const deadline = Date.now() + leaseDurationMs * 4; + let successorToken = null; + while (Date.now() < deadline) { + successorToken = await successor.acquire(); + if (successorToken) { + break; + } + await delay(100); + } + expect(successorToken).toMatchObject({ + profile, + ownerId: 'successor-owner', + fencingEpoch: 2n, + }); + expect(await successor.assertActive()).toBeUndefined(); + } finally { + if (holder.exitCode === null && holder.signalCode === null) { + holder.kill('SIGKILL'); + await once(holder, 'exit'); + } + await successor?.release(); + } + }, 15_000); }); diff --git a/tools/run-conditional-integration.sh b/tools/run-conditional-integration.sh index 2312be1..c43e59a 100755 --- a/tools/run-conditional-integration.sh +++ b/tools/run-conditional-integration.sh @@ -91,6 +91,7 @@ cd "$workspace_root" pnpm install --frozen-lockfile pnpm --filter @sammo-ts/infra prisma:generate pnpm --filter @sammo-ts/common build +pnpm --filter @sammo-ts/infra build database_url=$(build_database_url "$integration_schema") export POSTGRES_SCHEMA=$integration_schema From d5639bf4be349cf0a079f91df1cfbbdec914b9e4 Mon Sep 17 00:00:00 2001 From: hided62 Date: Mon, 27 Jul 2026 00:53:33 +0000 Subject: [PATCH 4/6] test turn daemon database partition fencing --- .../test/turnDaemonLease.integration.test.ts | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/app/game-engine/test/turnDaemonLease.integration.test.ts b/app/game-engine/test/turnDaemonLease.integration.test.ts index b4d7000..4531ad6 100644 --- a/app/game-engine/test/turnDaemonLease.integration.test.ts +++ b/app/game-engine/test/turnDaemonLease.integration.test.ts @@ -1,5 +1,6 @@ import { spawn } from 'node:child_process'; import { once } from 'node:events'; +import { createConnection, createServer, type Server, type Socket } from 'node:net'; import { fileURLToPath } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; @@ -47,6 +48,94 @@ const delay = (durationMs: number): Promise => setTimeout(resolve, durationMs); }); +const createPartitionableTcpProxy = async ( + upstreamHost: string, + upstreamPort: number +): Promise<{ + port: number; + partition: () => Promise; + restore: () => Promise; + close: () => Promise; +}> => { + const sockets = new Set(); + let server: Server | null = null; + let port = 0; + + const listen = async (requestedPort: number): Promise => { + const nextServer = createServer((client) => { + const upstream = createConnection({ host: upstreamHost, port: upstreamPort }); + sockets.add(client); + sockets.add(upstream); + client.on('error', () => undefined); + upstream.on('error', () => undefined); + client.once('close', () => { + sockets.delete(client); + if (!upstream.destroyed) { + upstream.destroy(); + } + }); + upstream.once('close', () => { + sockets.delete(upstream); + if (!client.destroyed) { + client.destroy(); + } + }); + client.pipe(upstream); + upstream.pipe(client); + }); + await new Promise((resolve, reject) => { + const onError = (error: Error): void => reject(error); + nextServer.once('error', onError); + nextServer.listen(requestedPort, '127.0.0.1', () => { + nextServer.off('error', onError); + resolve(); + }); + }); + server = nextServer; + const address = nextServer.address(); + if (!address || typeof address === 'string') { + throw new Error('TCP proxy did not expose a numeric port'); + } + port = address.port; + }; + + const partition = async (): Promise => { + const activeServer = server; + server = null; + if (!activeServer) { + return; + } + const closed = new Promise((resolve, reject) => { + activeServer.close((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + for (const socket of sockets) { + socket.destroy(); + } + sockets.clear(); + await closed; + }; + + await listen(0); + return { + get port() { + return port; + }, + partition, + restore: async () => { + if (!server) { + await listen(port); + } + }, + close: partition, + }; +}; + integration('database turn daemon lease and fencing', () => { let db: GamePrismaClient; let disconnect: (() => Promise) | undefined; @@ -211,4 +300,58 @@ integration('database turn daemon lease and fencing', () => { await successor?.release(); } }, 15_000); + + it('stays fenced after a database partition outlasts the heartbeat lease and connectivity recovers', async () => { + const profile = `${profilePrefix}database-partition`; + const leaseDurationMs = 1_800; + const directUrl = new URL(databaseUrl!); + const upstreamPort = Number(directUrl.port || '5432'); + const proxy = await createPartitionableTcpProxy(directUrl.hostname, upstreamPort); + directUrl.hostname = '127.0.0.1'; + directUrl.port = String(proxy.port); + const partitionedOwner = await DatabaseTurnDaemonLease.connect(directUrl.toString(), { + profile, + ownerId: 'partitioned-owner', + leaseDurationMs, + heartbeat: true, + }); + leases.push(partitionedOwner); + let successor: DatabaseTurnDaemonLease | null = null; + try { + expect((await partitionedOwner.acquire())?.fencingEpoch).toBe(1n); + + await proxy.partition(); + const lostDeadline = Date.now() + leaseDurationMs * 2; + while (!partitionedOwner.isLost() && Date.now() < lostDeadline) { + await delay(50); + } + expect(partitionedOwner.isLost()).toBe(true); + + successor = await createLease(profile, 'partition-successor'); + expect(await successor.acquire()).toBeNull(); + + const takeoverDeadline = Date.now() + leaseDurationMs * 4; + let successorToken = null; + while (Date.now() < takeoverDeadline) { + successorToken = await successor.acquire(); + if (successorToken) { + break; + } + await delay(100); + } + expect(successorToken).toMatchObject({ + profile, + ownerId: 'partition-successor', + fencingEpoch: 2n, + }); + + await proxy.restore(); + expect(await partitionedOwner.renew()).toBe(false); + await expect(partitionedOwner.assertActive()).rejects.toBeInstanceOf(TurnDaemonLeaseLostError); + await expect(successor.assertActive()).resolves.toBeUndefined(); + } finally { + await proxy.close(); + await successor?.release(); + } + }, 20_000); }); From 606fe268bd8865725d180d3bc3e6447b4c20c193 Mon Sep 17 00:00:00 2001 From: hided62 Date: Mon, 27 Jul 2026 00:58:47 +0000 Subject: [PATCH 5/6] fix lease fencing during silent database partitions --- .../src/lifecycle/databaseTurnDaemonLease.ts | 69 ++++++++++++++----- .../test/turnDaemonLease.integration.test.ts | 60 +++++++++++----- 2 files changed, 94 insertions(+), 35 deletions(-) diff --git a/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts b/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts index a40fe2d..b4eeba3 100644 --- a/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts +++ b/app/game-engine/src/lifecycle/databaseTurnDaemonLease.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; import { createGamePostgresConnector, GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; @@ -46,6 +47,8 @@ export class DatabaseTurnDaemonLease { private readonly heartbeatEnabled: boolean; private token: TurnDaemonLeaseToken | null = null; private heartbeatTimer: NodeJS.Timeout | null = null; + private expiryTimer: NodeJS.Timeout | null = null; + private renewalInFlight = false; private lost = false; private constructor( @@ -71,6 +74,7 @@ export class DatabaseTurnDaemonLease { } async acquire(): Promise { + const requestStartedAt = performance.now(); const rows = await this.db.$queryRaw(GamePrisma.sql` INSERT INTO "turn_daemon_lease" ( "profile", @@ -111,6 +115,7 @@ export class DatabaseTurnDaemonLease { fencingEpoch: BigInt(row.fencing_epoch), }; this.lost = false; + this.scheduleExpiryWatchdog(requestStartedAt); if (this.heartbeatEnabled) { this.startHeartbeat(); } @@ -127,26 +132,36 @@ export class DatabaseTurnDaemonLease { async renew(): Promise { const token = this.token; - if (!token || this.lost) { + if (!token || this.lost || this.renewalInFlight) { return false; } - const rows = await this.db.$queryRaw(GamePrisma.sql` - UPDATE "turn_daemon_lease" - SET - "lease_until" = CURRENT_TIMESTAMP + (${this.leaseDurationMs} * INTERVAL '1 millisecond'), - "heartbeat_at" = CURRENT_TIMESTAMP - WHERE - "profile" = ${token.profile} - AND "owner_id" = ${token.ownerId} - AND "fencing_epoch" = ${token.fencingEpoch} - AND "lease_until" > CURRENT_TIMESTAMP - RETURNING "profile", "owner_id", "fencing_epoch" - `); - if (rows.length === 0) { - this.markLost(); - return false; + const requestStartedAt = performance.now(); + this.renewalInFlight = true; + try { + const rows = await this.db.$queryRaw(GamePrisma.sql` + UPDATE "turn_daemon_lease" + SET + "lease_until" = CURRENT_TIMESTAMP + (${this.leaseDurationMs} * INTERVAL '1 millisecond'), + "heartbeat_at" = CURRENT_TIMESTAMP + WHERE + "profile" = ${token.profile} + AND "owner_id" = ${token.ownerId} + AND "fencing_epoch" = ${token.fencingEpoch} + AND "lease_until" > CURRENT_TIMESTAMP + RETURNING "profile", "owner_id", "fencing_epoch" + `); + if (rows.length === 0) { + this.markLost(); + return false; + } + if (this.lost) { + return false; + } + this.scheduleExpiryWatchdog(requestStartedAt); + return true; + } finally { + this.renewalInFlight = false; } - return true; } async assertActive(transaction?: GamePrisma.TransactionClient): Promise { @@ -173,6 +188,7 @@ export class DatabaseTurnDaemonLease { async release(): Promise { this.stopHeartbeat(); + this.stopExpiryWatchdog(); const token = this.token; this.token = null; if (!token || this.lost) { @@ -216,8 +232,27 @@ export class DatabaseTurnDaemonLease { } } + private scheduleExpiryWatchdog(requestStartedAt: number): void { + // DB가 lease_until을 정하는 시점보다 앞선 요청 시작 시각을 기준으로 + // 잡아, heartbeat 응답이 멈춰도 DB lease 만료보다 늦게 pause하지 않는다. + this.stopExpiryWatchdog(); + const remainingMs = Math.max(0, this.leaseDurationMs - (performance.now() - requestStartedAt)); + this.expiryTimer = setTimeout(() => { + this.markLost(); + }, remainingMs); + this.expiryTimer.unref(); + } + + private stopExpiryWatchdog(): void { + if (this.expiryTimer) { + clearTimeout(this.expiryTimer); + this.expiryTimer = null; + } + } + private markLost(): void { this.lost = true; this.stopHeartbeat(); + this.stopExpiryWatchdog(); } } diff --git a/app/game-engine/test/turnDaemonLease.integration.test.ts b/app/game-engine/test/turnDaemonLease.integration.test.ts index 4531ad6..540620c 100644 --- a/app/game-engine/test/turnDaemonLease.integration.test.ts +++ b/app/game-engine/test/turnDaemonLease.integration.test.ts @@ -57,31 +57,43 @@ const createPartitionableTcpProxy = async ( restore: () => Promise; close: () => Promise; }> => { - const sockets = new Set(); + type ProxyPair = { + client: Socket; + upstream: Socket; + }; + const pairs = new Set(); let server: Server | null = null; let port = 0; + let partitioned = false; + + const resumePair = (pair: ProxyPair): void => { + if (partitioned || pair.client.destroyed || pair.upstream.destroyed || pair.upstream.connecting) { + return; + } + pair.client.pipe(pair.upstream); + pair.upstream.pipe(pair.client); + }; const listen = async (requestedPort: number): Promise => { const nextServer = createServer((client) => { const upstream = createConnection({ host: upstreamHost, port: upstreamPort }); - sockets.add(client); - sockets.add(upstream); + const pair = { client, upstream }; + pairs.add(pair); client.on('error', () => undefined); upstream.on('error', () => undefined); client.once('close', () => { - sockets.delete(client); + pairs.delete(pair); if (!upstream.destroyed) { upstream.destroy(); } }); upstream.once('close', () => { - sockets.delete(upstream); + pairs.delete(pair); if (!client.destroyed) { client.destroy(); } }); - client.pipe(upstream); - upstream.pipe(client); + upstream.once('connect', () => resumePair(pair)); }); await new Promise((resolve, reject) => { const onError = (error: Error): void => reject(error); @@ -99,7 +111,16 @@ const createPartitionableTcpProxy = async ( port = address.port; }; - const partition = async (): Promise => { + const partition = (): Promise => { + partitioned = true; + for (const pair of pairs) { + pair.client.unpipe(pair.upstream); + pair.upstream.unpipe(pair.client); + } + return Promise.resolve(); + }; + + const close = async (): Promise => { const activeServer = server; server = null; if (!activeServer) { @@ -114,10 +135,11 @@ const createPartitionableTcpProxy = async ( } }); }); - for (const socket of sockets) { - socket.destroy(); + for (const pair of pairs) { + pair.client.destroy(); + pair.upstream.destroy(); } - sockets.clear(); + pairs.clear(); await closed; }; @@ -127,12 +149,14 @@ const createPartitionableTcpProxy = async ( return port; }, partition, - restore: async () => { - if (!server) { - await listen(port); + restore: () => { + partitioned = false; + for (const pair of pairs) { + resumePair(pair); } + return Promise.resolve(); }, - close: partition, + close, }; }; @@ -321,15 +345,15 @@ integration('database turn daemon lease and fencing', () => { expect((await partitionedOwner.acquire())?.fencingEpoch).toBe(1n); await proxy.partition(); + successor = await createLease(profile, 'partition-successor'); + expect(await successor.acquire()).toBeNull(); + const lostDeadline = Date.now() + leaseDurationMs * 2; while (!partitionedOwner.isLost() && Date.now() < lostDeadline) { await delay(50); } expect(partitionedOwner.isLost()).toBe(true); - successor = await createLease(profile, 'partition-successor'); - expect(await successor.acquire()).toBeNull(); - const takeoverDeadline = Date.now() + leaseDurationMs * 4; let successorToken = null; while (Date.now() < takeoverDeadline) { From bb0abcd3dec59eca21645f6aba2cba6cbb246ce1 Mon Sep 17 00:00:00 2001 From: hided62 Date: Mon, 27 Jul 2026 01:07:45 +0000 Subject: [PATCH 6/6] test admin security over HTTP transport --- .../test/adminSecurityTransport.e2e.test.ts | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 app/gateway-api/test/adminSecurityTransport.e2e.test.ts diff --git a/app/gateway-api/test/adminSecurityTransport.e2e.test.ts b/app/gateway-api/test/adminSecurityTransport.e2e.test.ts new file mode 100644 index 0000000..1a23f54 --- /dev/null +++ b/app/gateway-api/test/adminSecurityTransport.e2e.test.ts @@ -0,0 +1,320 @@ +import fastify, { type FastifyRequest } from 'fastify'; +import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify'; +import { afterEach, describe, expect, it } from 'vitest'; + +import type { GatewayPrismaClient } from '@sammo-ts/infra'; + +import { InMemoryGatewaySessionService } from '../src/auth/inMemorySessionService.js'; +import { createInMemoryUserRepository } from '../src/auth/inMemoryUserRepository.js'; +import { createPasswordEnvelopeService } from '../src/auth/passwordEnvelope.js'; +import { createGatewayApiContext } from '../src/context.js'; +import { InMemoryProfileStatusService } from '../src/lobby/profileStatusService.js'; +import type { GatewayProfileRepository } from '../src/orchestrator/profileRepository.js'; +import { appRouter } from '../src/router.js'; + +const profile = { + profileName: 'che:default', + profile: 'che', + scenario: 'default', + apiPort: 15003, + status: 'RUNNING' as const, + buildStatus: 'SUCCEEDED' as const, + meta: {}, + createdAt: '2026-07-26T00:00:00.000Z', + updatedAt: '2026-07-26T00:00:00.000Z', +}; + +const profiles: GatewayProfileRepository = { + listProfiles: async () => [profile], + getProfile: async (profileName) => (profileName === profile.profileName ? profile : null), + upsertProfile: async () => profile, + updateScenario: async () => profile, + updateStatus: async () => profile, + updateBuildStatus: async () => profile, + updateMeta: async () => profile, + listReservedToStart: async () => [], + findQueuedBuild: async () => null, + updateLastError: async () => {}, + updateWorkspaceUsage: async () => {}, + clearWorkspaceUsage: async () => {}, + listOperations: async () => [], + getOperation: async () => null, + createOperation: async () => { + throw new Error('not used'); + }, + claimNextOperation: async () => null, + completeOperation: async () => { + throw new Error('not used'); + }, + requeueOperation: async () => { + throw new Error('not used'); + }, + cancelOperation: async () => false, + retryOperation: async () => null, +}; + +const openApps = new Set>(); + +afterEach(async () => { + await Promise.allSettled(Array.from(openApps, (app) => app.close())); + openApps.clear(); +}); + +const createHarness = async (adminRoles = ['user', 'admin.users.manage', 'admin.survey.open:che:default']) => { + const users = createInMemoryUserRepository(); + const admin = await users.createUser({ + username: 'scoped-admin', + password: 'secretpass', + displayName: 'Scoped Admin', + }); + await users.updateRoles(admin.id, adminRoles); + const target = await users.createUser({ + username: 'target-user', + password: 'secretpass', + displayName: 'Target', + }); + const sessions = new InMemoryGatewaySessionService({ + sessionTtlSeconds: 600, + gameSessionTtlSeconds: 600, + }); + const adminSession = await sessions.createSession({ ...admin, roles: adminRoles }); + const targetSession = await sessions.createSession(target); + const flushes: Array<{ userId: string; reason?: string }> = []; + const app = fastify({ logger: false }); + await app.register(fastifyTRPCPlugin, { + prefix: '/trpc', + trpcOptions: { + router: appRouter, + createContext: ({ req }: { req: FastifyRequest }) => + createGatewayApiContext({ + users, + sessions, + flushPublisher: { + publishUserFlush: async (userId, reason) => { + flushes.push({ userId, reason }); + }, + }, + gameTokenSecret: 'transport-e2e-secret', + gameSessionTtlSeconds: 600, + kakaoClient: {} as never, + oauthSessions: {} as never, + publicBaseUrl: 'http://127.0.0.1', + adminLocalAccountEnabled: false, + localRegistrationEnabled: true, + localAccountGraceDays: 7, + passwordEnvelope: createPasswordEnvelopeService(), + profiles, + orchestrator: { + start: () => {}, + stop: async () => {}, + reconcileNow: async () => {}, + runScheduleNow: async () => {}, + runBuildQueueNow: async () => {}, + runOperationsNow: async () => {}, + cleanupStaleWorkspaces: async () => ({ removed: [], skipped: [] }), + listRuntimeStates: async () => [], + }, + profileStatus: new InMemoryProfileStatusService(), + requestHeaders: req.headers, + prisma: { + appUser: { + findFirst: async () => ({ id: 'bootstrap-user' }), + }, + } as unknown as GatewayPrismaClient, + }), + }, + }); + const baseUrl = await app.listen({ host: '127.0.0.1', port: 0 }); + openApps.add(app); + return { + app, + baseUrl, + users, + admin, + target, + adminSessionToken: adminSession.sessionToken, + targetSessionToken: targetSession.sessionToken, + flushes, + }; +}; + +const postTrpc = async ( + baseUrl: string, + procedure: string, + input: unknown, + sessionToken?: string +): Promise<{ response: Response; body: unknown }> => { + const response = await fetch(`${baseUrl}/trpc/${procedure}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(sessionToken ? { 'x-session-token': sessionToken } : {}), + }, + body: JSON.stringify(input), + }); + return { + response, + body: (await response.json()) as unknown, + }; +}; + +describe('admin security over HTTP transport', () => { + it('accepts an equal scoped role and rejects wildcard escalation without mutating roles', async () => { + const harness = await createHarness(); + + const allowed = await postTrpc( + harness.baseUrl, + 'admin.users.updateRoles', + { + userId: harness.target.id, + roles: ['admin.survey.open:che:default'], + mode: 'grant', + }, + harness.adminSessionToken + ); + if (allowed.response.status !== 200) { + throw new Error(`allowed role request failed: ${JSON.stringify(allowed.body)}`); + } + expect(allowed.body).toMatchObject({ + result: { + data: { + roles: ['user', 'admin.survey.open:che:default'], + }, + }, + }); + + const rejected = await postTrpc( + harness.baseUrl, + 'admin.users.updateRoles', + { + userId: harness.target.id, + roles: ['admin.survey.open:*'], + mode: 'grant', + }, + harness.adminSessionToken + ); + expect(rejected.response.status).toBe(403); + expect(rejected.body).toMatchObject({ + error: { + data: { + code: 'FORBIDDEN', + }, + }, + }); + expect((await harness.users.findById(harness.target.id))?.roles).toEqual([ + 'user', + 'admin.survey.open:che:default', + ]); + }); + + it('rejects an unauthenticated role change at the HTTP header boundary', async () => { + const harness = await createHarness(); + + const rejected = await postTrpc(harness.baseUrl, 'admin.users.updateRoles', { + userId: harness.target.id, + roles: ['admin.survey.open:che:default'], + mode: 'grant', + }); + + expect(rejected.response.status).toBe(401); + expect(rejected.body).toMatchObject({ + error: { + data: { + code: 'UNAUTHORIZED', + }, + }, + }); + expect((await harness.users.findById(harness.target.id))?.roles).toEqual(['user']); + }); + + it('rejects self-escalation and set-mode removal outside a scoped administrator role', async () => { + const harness = await createHarness(); + + const selfEscalation = await postTrpc( + harness.baseUrl, + 'admin.users.updateRoles', + { + userId: harness.admin.id, + roles: ['admin.survey.open:*'], + mode: 'grant', + }, + harness.adminSessionToken + ); + expect(selfEscalation.response.status).toBe(403); + expect((await harness.users.findByUsername('scoped-admin'))?.roles).toEqual([ + 'user', + 'admin.users.manage', + 'admin.survey.open:che:default', + ]); + + await harness.users.updateRoles(harness.target.id, ['user', 'admin.survey.open:*']); + const outOfScopeRemoval = await postTrpc( + harness.baseUrl, + 'admin.users.updateRoles', + { + userId: harness.target.id, + roles: ['user'], + mode: 'set', + }, + harness.adminSessionToken + ); + expect(outOfScopeRemoval.response.status).toBe(403); + expect((await harness.users.findById(harness.target.id))?.roles).toEqual(['user', 'admin.survey.open:*']); + }); + + it('allows a superuser to grant a root role over HTTP', async () => { + const harness = await createHarness(['user', 'superuser']); + + const granted = await postTrpc( + harness.baseUrl, + 'admin.users.updateRoles', + { + userId: harness.target.id, + roles: ['superuser'], + mode: 'grant', + }, + harness.adminSessionToken + ); + + expect(granted.response.status).toBe(200); + expect((await harness.users.findById(harness.target.id))?.roles).toEqual(['user', 'superuser']); + }); + + it('blocks game-session issuance after an HTTP sanction update while retaining the gateway session', async () => { + const harness = await createHarness(); + + const updated = await postTrpc( + harness.baseUrl, + 'admin.users.updateSanctions', + { + userId: harness.target.id, + patch: { + suspendedUntil: '2099-01-01T00:00:00.000Z', + }, + }, + harness.adminSessionToken + ); + if (updated.response.status !== 200) { + throw new Error(`sanction request failed: ${JSON.stringify(updated.body)}`); + } + expect(harness.flushes).toEqual([ + { + userId: harness.target.id, + reason: 'admin-sanctions-updated', + }, + ]); + + const blocked = await postTrpc(harness.baseUrl, 'auth.issueGameSession', { + sessionToken: harness.targetSessionToken, + profile: profile.profileName, + }); + expect(blocked.response.status).toBe(403); + expect(blocked.body).toMatchObject({ + error: { + data: { + code: 'FORBIDDEN', + }, + }, + }); + }); +});