From bff7b05c052f52be7e1416e54ebb18ac29c62de9 Mon Sep 17 00:00:00 2001 From: hided62 Date: Mon, 27 Jul 2026 00:46:22 +0000 Subject: [PATCH] test turn daemon process kill takeover --- .../test/helpers/turnDaemonLeaseHolder.mjs | 31 ++++++ .../test/turnDaemonLease.integration.test.ts | 96 +++++++++++++++++++ tools/run-conditional-integration.sh | 1 + 3 files changed, 128 insertions(+) create mode 100644 app/game-engine/test/helpers/turnDaemonLeaseHolder.mjs diff --git a/app/game-engine/test/helpers/turnDaemonLeaseHolder.mjs b/app/game-engine/test/helpers/turnDaemonLeaseHolder.mjs new file mode 100644 index 0000000..c4b253b --- /dev/null +++ b/app/game-engine/test/helpers/turnDaemonLeaseHolder.mjs @@ -0,0 +1,31 @@ +import { DatabaseTurnDaemonLease } from '../../src/lifecycle/databaseTurnDaemonLease.ts'; + +const databaseUrl = process.env.TURN_DAEMON_LEASE_DATABASE_URL; +const profile = process.env.TURN_DAEMON_LEASE_PROFILE; +const ownerId = process.env.TURN_DAEMON_LEASE_OWNER_ID; +const leaseDurationMs = Number(process.env.TURN_DAEMON_LEASE_DURATION_MS); + +if (!databaseUrl || !profile || !ownerId || !Number.isInteger(leaseDurationMs)) { + throw new Error('lease holder requires database URL, profile, owner, and duration'); +} + +const lease = await DatabaseTurnDaemonLease.connect(databaseUrl, { + profile, + ownerId, + leaseDurationMs, + heartbeat: true, +}); +const token = await lease.acquire(); +if (!token) { + throw new Error('lease holder could not acquire the profile lease'); +} + +process.stdout.write( + `${JSON.stringify({ + profile: token.profile, + ownerId: token.ownerId, + fencingEpoch: token.fencingEpoch.toString(), + })}\n` +); + +setInterval(() => undefined, 60_000); diff --git a/app/game-engine/test/turnDaemonLease.integration.test.ts b/app/game-engine/test/turnDaemonLease.integration.test.ts index e5c5ba7..b4d7000 100644 --- a/app/game-engine/test/turnDaemonLease.integration.test.ts +++ b/app/game-engine/test/turnDaemonLease.integration.test.ts @@ -1,3 +1,7 @@ +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { fileURLToPath } from 'node:url'; + import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra'; @@ -7,6 +11,41 @@ import { DatabaseTurnDaemonLease, TurnDaemonLeaseLostError } from '../src/lifecy const databaseUrl = process.env.TURN_DAEMON_LEASE_DATABASE_URL ?? process.env.INPUT_EVENT_DATABASE_URL; const integration = describe.skipIf(!databaseUrl); const profilePrefix = 'integration:turn-lease:'; +const holderScript = fileURLToPath(new URL('./helpers/turnDaemonLeaseHolder.mjs', import.meta.url)); + +type HolderReady = { + profile: string; + ownerId: string; + fencingEpoch: string; +}; + +const waitForHolderReady = async (child: ReturnType, timeoutMs = 10_000): Promise => { + let output = ''; + const ready = new Promise((resolve, reject) => { + child.stdout?.setEncoding('utf8'); + child.stdout?.on('data', (chunk: string) => { + output += chunk; + const newline = output.indexOf('\n'); + if (newline < 0) { + return; + } + resolve(JSON.parse(output.slice(0, newline)) as HolderReady); + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + reject(new Error(`lease holder exited before readiness: code=${String(code)} signal=${String(signal)}`)); + }); + }); + const timeout = new Promise((_, reject) => { + setTimeout(() => reject(new Error('lease holder readiness timed out')), timeoutMs).unref(); + }); + return Promise.race([ready, timeout]); +}; + +const delay = (durationMs: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, durationMs); + }); integration('database turn daemon lease and fencing', () => { let db: GamePrismaClient; @@ -115,4 +154,61 @@ integration('database turn daemon lease and fencing', () => { expect((await second.acquire())?.fencingEpoch).toBe(2n); await expect(first.assertActive()).rejects.toBeInstanceOf(TurnDaemonLeaseLostError); }); + + it('takes over after a heartbeat owner process is killed and its lease expires', async () => { + const profile = `${profilePrefix}process-kill`; + const leaseDurationMs = 1_200; + const holder = spawn(process.execPath, ['--experimental-strip-types', holderScript], { + env: { + ...process.env, + TURN_DAEMON_LEASE_DATABASE_URL: databaseUrl!, + TURN_DAEMON_LEASE_PROFILE: profile, + TURN_DAEMON_LEASE_OWNER_ID: 'process-owner', + TURN_DAEMON_LEASE_DURATION_MS: String(leaseDurationMs), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let successor: DatabaseTurnDaemonLease | null = null; + try { + await expect(waitForHolderReady(holder)).resolves.toEqual({ + profile, + ownerId: 'process-owner', + fencingEpoch: '1', + }); + expect(await db.turnDaemonLease.findUniqueOrThrow({ where: { profile } })).toMatchObject({ + ownerId: 'process-owner', + fencingEpoch: 1n, + }); + + expect(holder.kill('SIGKILL')).toBe(true); + const [exitCode, signal] = (await once(holder, 'exit')) as [number | null, NodeJS.Signals | null]; + expect(exitCode).toBeNull(); + expect(signal).toBe('SIGKILL'); + + successor = await createLease(profile, 'successor-owner'); + expect(await successor.acquire()).toBeNull(); + + const deadline = Date.now() + leaseDurationMs * 4; + let successorToken = null; + while (Date.now() < deadline) { + successorToken = await successor.acquire(); + if (successorToken) { + break; + } + await delay(100); + } + expect(successorToken).toMatchObject({ + profile, + ownerId: 'successor-owner', + fencingEpoch: 2n, + }); + expect(await successor.assertActive()).toBeUndefined(); + } finally { + if (holder.exitCode === null && holder.signalCode === null) { + holder.kill('SIGKILL'); + await once(holder, 'exit'); + } + await successor?.release(); + } + }, 15_000); }); diff --git a/tools/run-conditional-integration.sh b/tools/run-conditional-integration.sh index 2312be1..c43e59a 100755 --- a/tools/run-conditional-integration.sh +++ b/tools/run-conditional-integration.sh @@ -91,6 +91,7 @@ cd "$workspace_root" pnpm install --frozen-lockfile pnpm --filter @sammo-ts/infra prisma:generate pnpm --filter @sammo-ts/common build +pnpm --filter @sammo-ts/infra build database_url=$(build_database_url "$integration_schema") export POSTGRES_SCHEMA=$integration_schema