From 81e58a21bcfe03a96f6e146d27d001da38cf1018 Mon Sep 17 00:00:00 2001 From: hided62 Date: Fri, 31 Jul 2026 16:55:57 +0000 Subject: [PATCH] fix(gateway): make process shutdown idempotent --- .../src/lifecycle/shutdownController.ts | 56 ++++++++++++++ .../src/orchestrator/orchestratorServer.ts | 21 ++--- app/gateway-api/src/server.ts | 22 +++++- .../test/shutdownController.test.ts | 76 +++++++++++++++++++ docs/architecture/runtime.md | 7 ++ 5 files changed, 170 insertions(+), 12 deletions(-) create mode 100644 app/gateway-api/src/lifecycle/shutdownController.ts create mode 100644 app/gateway-api/test/shutdownController.test.ts diff --git a/app/gateway-api/src/lifecycle/shutdownController.ts b/app/gateway-api/src/lifecycle/shutdownController.ts new file mode 100644 index 0000000..3cead55 --- /dev/null +++ b/app/gateway-api/src/lifecycle/shutdownController.ts @@ -0,0 +1,56 @@ +export type GatewayShutdownReason = 'SIGINT' | 'SIGTERM' | string; + +export type GatewayShutdownController = { + stop(reason: GatewayShutdownReason): Promise; + dispose(): void; +}; + +type GatewayShutdownOptions = { + close(reason: GatewayShutdownReason): void | Promise; + onStopping?: (reason: GatewayShutdownReason) => void; + onError?: (error: unknown, reason: GatewayShutdownReason) => void; +}; + +/** Installs one idempotent owner for process signals and resource shutdown. */ +export const installGatewayShutdownController = (options: GatewayShutdownOptions): GatewayShutdownController => { + let stopPromise: Promise | undefined; + let stopReason: GatewayShutdownReason | undefined; + let failureReported = false; + let disposed = false; + + const dispose = (): void => { + if (disposed) return; + disposed = true; + process.off('SIGINT', handleSigint); + process.off('SIGTERM', handleSigterm); + }; + + const stop = (reason: GatewayShutdownReason): Promise => { + if (stopPromise) return stopPromise; + stopReason = reason; + options.onStopping?.(reason); + stopPromise = Promise.resolve() + .then(() => options.close(reason)) + .finally(dispose); + return stopPromise; + }; + + const requestStop = (reason: GatewayShutdownReason): void => { + void stop(reason).catch((error: unknown) => { + if (failureReported) return; + failureReported = true; + options.onError?.(error, stopReason ?? reason); + }); + }; + function handleSigint(): void { + requestStop('SIGINT'); + } + function handleSigterm(): void { + requestStop('SIGTERM'); + } + + process.on('SIGINT', handleSigint); + process.on('SIGTERM', handleSigterm); + + return { stop, dispose }; +}; diff --git a/app/gateway-api/src/orchestrator/orchestratorServer.ts b/app/gateway-api/src/orchestrator/orchestratorServer.ts index 6f8468f..6485e59 100644 --- a/app/gateway-api/src/orchestrator/orchestratorServer.ts +++ b/app/gateway-api/src/orchestrator/orchestratorServer.ts @@ -6,6 +6,7 @@ import { import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js'; import { createGatewayOrchestrator } from './orchestratorFactory.js'; +import { installGatewayShutdownController } from '../lifecycle/shutdownController.js'; export const runGatewayOrchestrator = async (): Promise => { const config = resolveGatewayOrchestratorConfigFromEnv(); @@ -14,15 +15,17 @@ export const runGatewayOrchestrator = async (): Promise => { const { orchestrator } = createGatewayOrchestrator(postgres.prisma as GatewayPrismaClient, config, process.env); - const stop = async (reason: string): Promise => { - console.info(`[gateway-orchestrator] stopping: ${reason}`); - await orchestrator.stop(); - await postgres.disconnect(); - }; - - process.on('SIGINT', () => void stop('SIGINT')); - process.on('SIGTERM', () => void stop('SIGTERM')); - orchestrator.start(); + installGatewayShutdownController({ + close: async () => { + await orchestrator.stop(); + await postgres.disconnect(); + }, + onStopping: (reason) => console.info(`[gateway-orchestrator] stopping: ${reason}`), + onError: (error, reason) => { + console.error(`[gateway-orchestrator] shutdown failed (${reason})`, error); + process.exitCode = 1; + }, + }); console.info('[gateway-orchestrator] started'); }; diff --git a/app/gateway-api/src/server.ts b/app/gateway-api/src/server.ts index ef86646..3e25649 100644 --- a/app/gateway-api/src/server.ts +++ b/app/gateway-api/src/server.ts @@ -25,6 +25,7 @@ import { createGatewayOrchestrator } from './orchestrator/orchestratorFactory.js import { appRouter } from './router.js'; import { RepositoryProfileStatusService } from './lobby/profileStatusService.js'; import { registerAccountIconInternalRoute } from './auth/accountIconInternalRoute.js'; +import { installGatewayShutdownController } from './lifecycle/shutdownController.js'; export const createGatewayApiServer = async () => { const config = resolveGatewayApiConfigFromEnv(); @@ -131,8 +132,23 @@ export const createGatewayApiServer = async () => { export const runGatewayApiServer = async (): Promise => { const { app, config } = await createGatewayApiServer(); - await app.listen({ - host: config.host, - port: config.port, + const shutdown = installGatewayShutdownController({ + close: () => app.close(), + onStopping: (reason) => app.log.info({ reason }, 'gateway API stopping'), + onError: (error, reason) => { + app.log.error({ err: error, reason }, 'gateway API shutdown failed'); + process.exitCode = 1; + }, }); + app.addHook('onClose', async () => shutdown.dispose()); + try { + await app.listen({ + host: config.host, + port: config.port, + }); + } catch (error) { + shutdown.dispose(); + await app.close(); + throw error; + } }; diff --git a/app/gateway-api/test/shutdownController.test.ts b/app/gateway-api/test/shutdownController.test.ts new file mode 100644 index 0000000..8751b15 --- /dev/null +++ b/app/gateway-api/test/shutdownController.test.ts @@ -0,0 +1,76 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { installGatewayShutdownController } from '../src/lifecycle/shutdownController.js'; + +const controllers: Array<{ dispose(): void }> = []; + +afterEach(() => { + for (const controller of controllers.splice(0)) controller.dispose(); + vi.restoreAllMocks(); +}); + +describe('gateway shutdown controller', () => { + it('drains resources once when SIGINT and SIGTERM arrive together', async () => { + const initialSigint = process.listenerCount('SIGINT'); + const initialSigterm = process.listenerCount('SIGTERM'); + let releaseClose = (): void => {}; + const closeGate = new Promise((resolve) => { + releaseClose = resolve; + }); + const close = vi.fn(async () => closeGate); + const onStopping = vi.fn(); + const controller = installGatewayShutdownController({ close, onStopping }); + controllers.push(controller); + + expect(process.listenerCount('SIGINT')).toBe(initialSigint + 1); + expect(process.listenerCount('SIGTERM')).toBe(initialSigterm + 1); + + process.emit('SIGINT'); + process.emit('SIGTERM'); + const manualStop = controller.stop('manual'); + await vi.waitFor(() => expect(close).toHaveBeenCalledTimes(1)); + releaseClose(); + await manualStop; + await controller.stop('later'); + + expect(close).toHaveBeenCalledTimes(1); + expect(onStopping).toHaveBeenCalledTimes(1); + expect(onStopping).toHaveBeenCalledWith('SIGINT'); + expect(process.listenerCount('SIGINT')).toBe(initialSigint); + expect(process.listenerCount('SIGTERM')).toBe(initialSigterm); + }); + + it('reports a close failure once and still removes both signal listeners', async () => { + const initialSigint = process.listenerCount('SIGINT'); + const initialSigterm = process.listenerCount('SIGTERM'); + const failure = new Error('disconnect failed'); + const onError = vi.fn(); + const controller = installGatewayShutdownController({ + close: () => { + throw failure; + }, + onError, + }); + controllers.push(controller); + + process.emit('SIGTERM'); + process.emit('SIGINT'); + await expect(controller.stop('manual')).rejects.toBe(failure); + await vi.waitFor(() => expect(onError).toHaveBeenCalledWith(failure, 'SIGTERM')); + + expect(onError).toHaveBeenCalledTimes(1); + expect(process.listenerCount('SIGINT')).toBe(initialSigint); + expect(process.listenerCount('SIGTERM')).toBe(initialSigterm); + }); + + it('can remove unused handlers without closing resources', () => { + const close = vi.fn(); + const controller = installGatewayShutdownController({ close }); + controllers.push(controller); + + controller.dispose(); + process.emit('SIGINT'); + + expect(close).not.toHaveBeenCalled(); + }); +}); diff --git a/docs/architecture/runtime.md b/docs/architecture/runtime.md index facb4b3..266722f 100644 --- a/docs/architecture/runtime.md +++ b/docs/architecture/runtime.md @@ -44,6 +44,13 @@ DB partial unique index로 한 건만 허용합니다. Turn daemon은 자신의 Redis 단계가 실패하면 action은 `PARTIAL`과 backoff 상태로 남고 DB 시간은 다시 이동하지 않습니다. +Gateway API와 독립 orchestrator의 SIGINT·SIGTERM은 +`installGatewayShutdownController()`가 하나의 종료 Promise로 합칩니다. +Gateway API는 Fastify `app.close()`를 통해 orchestrator task를 drain한 뒤 +Redis와 PostgreSQL을 닫습니다. 독립 orchestrator도 task drain 뒤 PostgreSQL을 +한 번만 닫습니다. 반복 signal, 종료 완료 뒤 재호출과 close 실패에서도 첫 +reason만 소유하고 등록한 signal listener를 해제합니다. + ## Game API 실행 `resolveGameApiConfigFromEnv()`가 `PROFILE`, `SCENARIO`,