feat: implement input event system with durable command handling

- Introduced InputEvent model with status tracking (PENDING, PROCESSING, SUCCEEDED, FAILED) and unique request IDs.
- Added DatabaseTurnDaemonTransport for sending commands and handling idempotency.
- Implemented executeInputEvent function to manage input event lifecycle and error handling.
- Created DatabaseTurnDaemonCommandQueue for managing command processing and lease recovery.
- Enhanced turn daemon lifecycle to support atomic command execution and error recovery.
- Added tests for input event atomicity, command queuing, and error handling scenarios.
This commit is contained in:
2026-07-25 05:36:34 +00:00
parent c5da507df9
commit 5040691d7c
34 changed files with 1587 additions and 448 deletions
+3
View File
@@ -61,6 +61,7 @@ export type InputJsonValue = GamePrisma.InputJsonValue;
export type DatabaseClient = InfraDatabaseClient;
export interface GameApiContext {
requestId?: string;
db: DatabaseClient;
redis: RedisConnector['client'];
turnDaemon: TurnDaemonTransport;
@@ -76,6 +77,7 @@ export interface GameApiContext {
}
export const createGameApiContext = (options: {
requestId?: string;
db: DatabaseClient;
redis: RedisConnector['client'];
turnDaemon: TurnDaemonTransport;
@@ -90,6 +92,7 @@ export const createGameApiContext = (options: {
gameTokenSecret: string;
}): GameApiContext => {
return {
requestId: options.requestId,
db: options.db,
redis: options.redis,
turnDaemon: options.turnDaemon,
@@ -0,0 +1,95 @@
import { randomUUID } from 'node:crypto';
import type { GamePrisma } from '@sammo-ts/infra';
import type { DatabaseClient } from '../context.js';
import type { TurnDaemonTransport } from './transport.js';
import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js';
const asJson = (value: unknown): GamePrisma.InputJsonValue => value as GamePrisma.InputJsonValue;
const delay = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
const stableJson = (value: unknown): string => {
if (Array.isArray(value)) {
return `[${value.map(stableJson).join(',')}]`;
}
if (value && typeof value === 'object') {
return `{${Object.entries(value)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`)
.join(',')}}`;
}
return JSON.stringify(value) ?? 'null';
};
export class ConflictingTurnDaemonCommandError extends Error {
constructor(readonly requestId: string) {
super(`Engine input event ${requestId} already exists with a different payload.`);
this.name = 'ConflictingTurnDaemonCommandError';
}
}
export class DatabaseTurnDaemonTransport implements TurnDaemonTransport {
constructor(
private readonly db: DatabaseClient,
private readonly requestTimeoutMs: number
) {}
async sendCommand(command: TurnDaemonCommand): Promise<string> {
const requestId = ('requestId' in command ? command.requestId : undefined) ?? randomUUID();
const durableCommand = JSON.parse(JSON.stringify({ ...command, requestId })) as TurnDaemonCommand;
try {
await this.db.inputEvent.create({
data: {
requestId,
target: 'ENGINE',
eventType: command.type,
payload: asJson(durableCommand),
},
});
} catch (error) {
const isUniqueConflict =
typeof error === 'object' && error !== null && 'code' in error && error.code === 'P2002';
if (!isUniqueConflict) {
throw error;
}
const existing = await this.db.inputEvent.findUniqueOrThrow({
where: { requestId },
select: { eventType: true, payload: true },
});
if (existing.eventType !== command.type || stableJson(existing.payload) !== stableJson(durableCommand)) {
throw new ConflictingTurnDaemonCommandError(requestId);
}
}
return requestId;
}
async requestCommand(command: TurnDaemonCommand, timeoutMs?: number): Promise<TurnDaemonCommandResult | null> {
const requestId = await this.sendCommand(command);
return this.waitForResult<TurnDaemonCommandResult>(requestId, timeoutMs);
}
async requestStatus(timeoutMs?: number): Promise<TurnDaemonStatus | null> {
const requestId = await this.sendCommand({ type: 'getStatus', requestId: randomUUID() });
const payload = await this.waitForResult<{ status: TurnDaemonStatus }>(requestId, timeoutMs);
return payload?.status ?? null;
}
private async waitForResult<T>(requestId: string, timeoutMs?: number): Promise<T | null> {
const deadline = Date.now() + (timeoutMs ?? this.requestTimeoutMs);
while (Date.now() < deadline) {
const event = await this.db.inputEvent.findUnique({
where: { requestId },
select: { status: true, result: true },
});
if (event?.status === 'SUCCEEDED') {
return event.result as T;
}
if (event?.status === 'FAILED') {
return null;
}
await delay(Math.min(50, Math.max(1, deadline - Date.now())));
}
return null;
}
}
@@ -0,0 +1,28 @@
import type { TurnDaemonTransport } from './transport.js';
import type { TurnDaemonCommand, TurnDaemonCommandResult, TurnDaemonStatus } from './types.js';
export class IdempotentTurnDaemonTransport implements TurnDaemonTransport {
private sequence = 0;
constructor(
private readonly transport: TurnDaemonTransport,
private readonly parentRequestId: string
) {}
async sendCommand(command: TurnDaemonCommand): Promise<string> {
return this.transport.sendCommand(this.scope(command));
}
async requestCommand(command: TurnDaemonCommand, timeoutMs?: number): Promise<TurnDaemonCommandResult | null> {
return this.transport.requestCommand(this.scope(command), timeoutMs);
}
async requestStatus(timeoutMs?: number): Promise<TurnDaemonStatus | null> {
return this.transport.requestStatus(timeoutMs);
}
private scope(command: TurnDaemonCommand): TurnDaemonCommand {
const requestId = `${this.parentRequestId}:engine:${this.sequence++}:${command.type}`;
return { ...command, requestId } as TurnDaemonCommand;
}
}
+1 -1
View File
@@ -27,7 +27,7 @@ export class InMemoryTurnDaemonTransport implements TurnDaemonTransport {
// 테스트용: 메모리 큐에 명령을 저장하고 requestId를 반환한다.
async sendCommand(command: TurnDaemonCommand): Promise<string> {
const requestId = command.type === 'getStatus' && command.requestId ? command.requestId : randomUUID();
const requestId = command.requestId ?? randomUUID();
this.commands.push({
requestId,
sentAt: new Date().toISOString(),
+1 -1
View File
@@ -26,7 +26,7 @@ type RedisStreamReadResponse = Array<{
}>;
const buildCommandEnvelope = (command: TurnDaemonCommand): TurnDaemonCommandEnvelope => {
const requestId = command.type === 'getStatus' && command.requestId ? command.requestId : randomUUID();
const requestId = command.requestId ?? randomUUID();
return {
requestId,
sentAt: new Date().toISOString(),
+7 -4
View File
@@ -8,11 +8,14 @@ import { runTournamentWorker } from './tournament/worker.js';
export * from './config.js';
export * from './context.js';
export * from './inputEventBoundary.js';
export * from './router.js';
export * from './server.js';
export * from './daemon/types.js';
export * from './daemon/streamKeys.js';
export * from './daemon/transport.js';
export * from './daemon/databaseTransport.js';
export * from './daemon/idempotentTransport.js';
export * from './daemon/inMemoryTransport.js';
export * from './daemon/redisTransport.js';
export * from './auth/flushStore.js';
@@ -51,10 +54,10 @@ if (isMain()) {
role === 'battle-sim-worker'
? runBattleSimWorker
: role === 'auction-worker'
? runAuctionWorker
: role === 'tournament-worker'
? runTournamentWorker
: runGameApiServer;
? runAuctionWorker
: role === 'tournament-worker'
? runTournamentWorker
: runGameApiServer;
run().catch((error) => {
console.error('[game-api] failed to start', error);
process.exitCode = 1;
+86
View File
@@ -0,0 +1,86 @@
import type { GamePrisma } from '@sammo-ts/infra';
import type { DatabaseClient } from './context.js';
const asJson = (value: unknown): GamePrisma.InputJsonValue => value as GamePrisma.InputJsonValue;
export class DuplicateInputEventError extends Error {
constructor(readonly requestId: string) {
super(`Input event ${requestId} was already accepted.`);
this.name = 'DuplicateInputEventError';
}
}
export const executeInputEvent = async <T>(options: {
db: DatabaseClient;
requestId: string;
eventType: string;
actorUserId?: string | null;
execute(db: DatabaseClient): Promise<T>;
}): Promise<T> => {
const { db, requestId, eventType, actorUserId, execute } = options;
if (!db.$transaction) {
return execute(db);
}
const processingAt = new Date();
try {
await db.inputEvent.create({
data: {
requestId,
target: 'API',
eventType,
payload: asJson({}),
actorUserId: actorUserId ?? null,
status: 'PROCESSING',
processingAt,
attempts: 1,
},
});
} catch (error) {
const isUniqueConflict =
typeof error === 'object' && error !== null && 'code' in error && error.code === 'P2002';
if (!isUniqueConflict) {
throw error;
}
const claimedRetry = await db.inputEvent.updateMany({
where: { requestId, status: 'FAILED' },
data: {
status: 'PROCESSING',
error: null,
processingAt,
completedAt: null,
attempts: { increment: 1 },
},
});
if (claimedRetry.count === 0) {
throw new DuplicateInputEventError(requestId);
}
}
try {
return await db.$transaction(async (transaction) => {
const result = await execute(transaction);
await transaction.inputEvent.update({
where: { requestId },
data: {
status: 'SUCCEEDED',
result: asJson({ ok: true }),
completedAt: new Date(),
},
});
return result;
});
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown API input event error.';
await db.inputEvent.update({
where: { requestId },
data: {
status: 'FAILED',
error: message,
completedAt: new Date(),
},
});
throw error;
}
};
+7 -10
View File
@@ -14,8 +14,7 @@ import {
import { resolveGameApiConfigFromEnv } from './config.js';
import { createGameApiContext, type DatabaseClient as _DatabaseClient } from './context.js';
import { buildTurnDaemonStreamKeys } from './daemon/streamKeys.js';
import { RedisTurnDaemonTransport } from './daemon/redisTransport.js';
import { DatabaseTurnDaemonTransport } from './daemon/databaseTransport.js';
import { InMemoryFlushStore, RedisGatewayFlushSubscriber, type FlushStore } from './auth/flushStore.js';
import { RedisAccessTokenStore } from './auth/accessTokenStore.js';
import { appRouter } from './router.js';
@@ -66,10 +65,7 @@ export const createGameApiServer = async () => {
await postgres.connect();
await redis.connect();
const turnDaemon = new RedisTurnDaemonTransport(redis.client, {
keys: buildTurnDaemonStreamKeys(config.profileName),
requestTimeoutMs: config.daemonRequestTimeoutMs,
});
const turnDaemon = new DatabaseTurnDaemonTransport(postgres.prisma, config.daemonRequestTimeoutMs);
const battleSim = new RedisBattleSimTransport(redis.client, {
keys: buildBattleSimQueueKeys(config.profileName),
requestTimeoutMs: config.battleSimRequestTimeoutMs,
@@ -83,10 +79,7 @@ export const createGameApiServer = async () => {
const accessTokenStore = new RedisAccessTokenStore(redis.client, config.profileName);
const realtimeSubscriberClient = redis.client.duplicate();
await realtimeSubscriberClient.connect();
const realtimeHub = new RedisRealtimeEventHub(
realtimeSubscriberClient,
buildGameEventChannel(config.profileName)
);
const realtimeHub = new RedisRealtimeEventHub(realtimeSubscriberClient, buildGameEventChannel(config.profileName));
await realtimeHub.start();
const app = fastify({
@@ -111,6 +104,10 @@ export const createGameApiServer = async () => {
const token = extractBearerToken(req.headers.authorization);
const auth = await resolveAuthFromToken(token, accessTokenStore, flushStore);
return createGameApiContext({
requestId:
(Array.isArray(req.headers['idempotency-key'])
? req.headers['idempotency-key'][0]
: req.headers['idempotency-key']) || undefined,
db: postgres.prisma,
redis: redis.client,
turnDaemon,
+42 -2
View File
@@ -1,12 +1,52 @@
import { randomUUID } from 'node:crypto';
import { initTRPC, TRPCError } from '@trpc/server';
import type { GameApiContext } from './context.js';
import { IdempotentTurnDaemonTransport } from './daemon/idempotentTransport.js';
import { DuplicateInputEventError, executeInputEvent } from './inputEventBoundary.js';
const t = initTRPC.context<GameApiContext>().create();
const inputEventMiddleware = t.middleware(async ({ ctx, type, path, next }) => {
if (type !== 'mutation' || !ctx.db.$transaction) {
return next();
}
const requestId = `${ctx.requestId ?? randomUUID()}:${path}`;
try {
return await executeInputEvent({
db: ctx.db,
requestId,
eventType: path,
actorUserId: ctx.auth?.user.id,
execute: async (transaction) => {
const result = await next({
ctx: {
...ctx,
db: transaction,
turnDaemon: new IdempotentTurnDaemonTransport(ctx.turnDaemon, requestId),
},
});
if (!result.ok) {
throw result.error;
}
return result;
},
});
} catch (error) {
if (error instanceof DuplicateInputEventError) {
throw new TRPCError({
code: 'CONFLICT',
message: error.message,
});
}
throw error;
}
});
export const router = t.router;
export const procedure = t.procedure;
export const authedProcedure: typeof t.procedure = t.procedure.use(({ ctx, next }) => {
export const procedure = t.procedure.use(inputEventMiddleware);
export const authedProcedure: typeof procedure = procedure.use(({ ctx, next }) => {
if (!ctx.auth) {
throw new TRPCError({
code: 'UNAUTHORIZED',
@@ -0,0 +1,22 @@
import { describe, expect, it } from 'vitest';
import { IdempotentTurnDaemonTransport } from '../src/daemon/idempotentTransport.js';
import { InMemoryTurnDaemonTransport } from '../src/daemon/inMemoryTransport.js';
describe('IdempotentTurnDaemonTransport', () => {
it('derives stable ordered engine request IDs from the API input event', async () => {
const inner = new InMemoryTurnDaemonTransport();
const firstAttempt = new IdempotentTurnDaemonTransport(inner, 'api-event');
const retry = new IdempotentTurnDaemonTransport(inner, 'api-event');
await firstAttempt.sendCommand({ type: 'vacation', generalId: 7 });
await firstAttempt.sendCommand({ type: 'dropItem', generalId: 7, itemType: 'weapon' });
await retry.sendCommand({ type: 'vacation', generalId: 7 });
expect(inner.commands.map((entry) => entry.requestId)).toEqual([
'api-event:engine:0:vacation',
'api-event:engine:1:dropItem',
'api-event:engine:0:vacation',
]);
});
});
@@ -0,0 +1,147 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createGamePostgresConnector, type GamePrismaClient } from '@sammo-ts/infra';
import { DuplicateInputEventError, executeInputEvent } from '../src/inputEventBoundary.js';
import { ConflictingTurnDaemonCommandError, DatabaseTurnDaemonTransport } from '../src/daemon/databaseTransport.js';
const databaseUrl = process.env.INPUT_EVENT_DATABASE_URL;
const integration = describe.skipIf(!databaseUrl);
integration('API input event boundary', () => {
let close: (() => Promise<void>) | undefined;
let db: GamePrismaClient;
beforeAll(async () => {
const connector = createGamePostgresConnector({ url: databaseUrl! });
await connector.connect();
db = connector.prisma;
close = () => connector.disconnect();
await db.inputEvent.deleteMany({
where: { requestId: { startsWith: 'integration:api:' } },
});
});
afterAll(async () => {
await db.inputEvent.deleteMany({
where: { requestId: { startsWith: 'integration:api:' } },
});
await close?.();
});
it('commits a direct DB mutation and its event marker together', async () => {
const requestId = 'integration:api:success';
const markerId = 'integration:api:success:marker';
await executeInputEvent({
db,
requestId,
eventType: 'test.success',
actorUserId: 'user-7',
execute: async (transaction) => {
await transaction.inputEvent.create({
data: {
requestId: markerId,
target: 'API',
eventType: 'test.marker',
},
});
return { ok: true };
},
});
const [event, marker] = await Promise.all([
db.inputEvent.findUniqueOrThrow({ where: { requestId } }),
db.inputEvent.findUniqueOrThrow({ where: { requestId: markerId } }),
]);
expect(event).toMatchObject({
status: 'SUCCEEDED',
actorUserId: 'user-7',
attempts: 1,
});
expect(marker.status).toBe('PENDING');
});
it('rolls back business writes, records failure, and permits one explicit retry', async () => {
const requestId = 'integration:api:retry';
const markerId = 'integration:api:retry:marker';
await expect(
executeInputEvent({
db,
requestId,
eventType: 'test.failure',
execute: async (transaction) => {
await transaction.inputEvent.create({
data: {
requestId: markerId,
target: 'API',
eventType: 'test.marker',
},
});
throw new Error('injected transaction failure');
},
})
).rejects.toThrow('injected transaction failure');
expect(await db.inputEvent.findUnique({ where: { requestId: markerId } })).toBeNull();
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId } })).toMatchObject({
status: 'FAILED',
attempts: 1,
error: 'injected transaction failure',
});
await executeInputEvent({
db,
requestId,
eventType: 'test.failure',
execute: async () => ({ ok: true }),
});
expect(await db.inputEvent.findUniqueOrThrow({ where: { requestId } })).toMatchObject({
status: 'SUCCEEDED',
attempts: 2,
});
});
it('rejects a concurrent duplicate idempotency key', async () => {
const requestId = 'integration:api:duplicate';
let releaseFirst: (() => void) | undefined;
let signalStarted: (() => void) | undefined;
const started = new Promise<void>((resolve) => {
signalStarted = resolve;
});
const release = new Promise<void>((resolve) => {
releaseFirst = resolve;
});
const first = executeInputEvent({
db,
requestId,
eventType: 'test.duplicate',
execute: async () => {
signalStarted?.();
await release;
return { ok: true };
},
});
await started;
await expect(
executeInputEvent({
db,
requestId,
eventType: 'test.duplicate',
execute: async () => ({ ok: true }),
})
).rejects.toBeInstanceOf(DuplicateInputEventError);
releaseFirst?.();
await first;
});
it('reuses the same engine child event but rejects a changed retry payload', async () => {
const transport = new DatabaseTurnDaemonTransport(db, 100);
const requestId = 'integration:api:engine-child';
await transport.sendCommand({ type: 'vacation', requestId, generalId: 7 });
await expect(transport.sendCommand({ type: 'vacation', requestId, generalId: 7 })).resolves.toBe(requestId);
await expect(transport.sendCommand({ type: 'vacation', requestId, generalId: 8 })).rejects.toBeInstanceOf(
ConflictingTurnDaemonCommandError
);
});
});