fix(gateway): make process shutdown idempotent

2026-07-31 16:55:57 +00:00
parent 1dd1610cb8
commit 81e58a21bc
5 changed files with 170 additions and 12 deletions
@@ -0,0 +1,56 @@
+export type GatewayShutdownReason = 'SIGINT' | 'SIGTERM' | string;
+
+export type GatewayShutdownController = {
+    stop(reason: GatewayShutdownReason): Promise<void>;
+    dispose(): void;
+};
+
+type GatewayShutdownOptions = {
+    close(reason: GatewayShutdownReason): void | Promise<void>;
+    onStopping?: (reason: GatewayShutdownReason) => void;
+    onError?: (error: unknown, reason: GatewayShutdownReason) => void;
+};
+
+/** Installs one idempotent owner for process signals and resource shutdown. */
+export const installGatewayShutdownController = (options: GatewayShutdownOptions): GatewayShutdownController => {
+    let stopPromise: Promise<void> | undefined;
+    let stopReason: GatewayShutdownReason | undefined;
+    let failureReported = false;
+    let disposed = false;
+
+    const dispose = (): void => {
+        if (disposed) return;
+        disposed = true;
+        process.off('SIGINT', handleSigint);
+        process.off('SIGTERM', handleSigterm);
+    };
+
+    const stop = (reason: GatewayShutdownReason): Promise<void> => {
+        if (stopPromise) return stopPromise;
+        stopReason = reason;
+        options.onStopping?.(reason);
+        stopPromise = Promise.resolve()
+            .then(() => options.close(reason))
+            .finally(dispose);
+        return stopPromise;
+    };
+
+    const requestStop = (reason: GatewayShutdownReason): void => {
+        void stop(reason).catch((error: unknown) => {
+            if (failureReported) return;
+            failureReported = true;
+            options.onError?.(error, stopReason ?? reason);
+        });
+    };
+    function handleSigint(): void {
+        requestStop('SIGINT');
+    }
+    function handleSigterm(): void {
+        requestStop('SIGTERM');
+    }
+
+    process.on('SIGINT', handleSigint);
+    process.on('SIGTERM', handleSigterm);
+
+    return { stop, dispose };
+};
@@ -6,6 +6,7 @@ import {

 import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js';
 import { createGatewayOrchestrator } from './orchestratorFactory.js';
+import { installGatewayShutdownController } from '../lifecycle/shutdownController.js';

 export const runGatewayOrchestrator = async (): Promise<void> => {
    const config = resolveGatewayOrchestratorConfigFromEnv();
@@ -14,15 +15,17 @@ export const runGatewayOrchestrator = async (): Promise<void> => {

    const { orchestrator } = createGatewayOrchestrator(postgres.prisma as GatewayPrismaClient, config, process.env);

-    const stop = async (reason: string): Promise<void> => {
-        console.info(`[gateway-orchestrator] stopping: ${reason}`);
-        await orchestrator.stop();
-        await postgres.disconnect();
-    };
-
-    process.on('SIGINT', () => void stop('SIGINT'));
-    process.on('SIGTERM', () => void stop('SIGTERM'));
-
    orchestrator.start();
+    installGatewayShutdownController({
+        close: async () => {
+            await orchestrator.stop();
+            await postgres.disconnect();
+        },
+        onStopping: (reason) => console.info(`[gateway-orchestrator] stopping: ${reason}`),
+        onError: (error, reason) => {
+            console.error(`[gateway-orchestrator] shutdown failed (${reason})`, error);
+            process.exitCode = 1;
+        },
+    });
    console.info('[gateway-orchestrator] started');
 };
@@ -25,6 +25,7 @@ import { createGatewayOrchestrator } from './orchestrator/orchestratorFactory.js
 import { appRouter } from './router.js';
 import { RepositoryProfileStatusService } from './lobby/profileStatusService.js';
 import { registerAccountIconInternalRoute } from './auth/accountIconInternalRoute.js';
+import { installGatewayShutdownController } from './lifecycle/shutdownController.js';

 export const createGatewayApiServer = async () => {
    const config = resolveGatewayApiConfigFromEnv();
@@ -131,8 +132,23 @@ export const createGatewayApiServer = async () => {

 export const runGatewayApiServer = async (): Promise<void> => {
    const { app, config } = await createGatewayApiServer();
-    await app.listen({
-        host: config.host,
-        port: config.port,
+    const shutdown = installGatewayShutdownController({
+        close: () => app.close(),
+        onStopping: (reason) => app.log.info({ reason }, 'gateway API stopping'),
+        onError: (error, reason) => {
+            app.log.error({ err: error, reason }, 'gateway API shutdown failed');
+            process.exitCode = 1;
+        },
    });
+    app.addHook('onClose', async () => shutdown.dispose());
+    try {
+        await app.listen({
+            host: config.host,
+            port: config.port,
+        });
+    } catch (error) {
+        shutdown.dispose();
+        await app.close();
+        throw error;
+    }
 };
@@ -0,0 +1,76 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+import { installGatewayShutdownController } from '../src/lifecycle/shutdownController.js';
+
+const controllers: Array<{ dispose(): void }> = [];
+
+afterEach(() => {
+    for (const controller of controllers.splice(0)) controller.dispose();
+    vi.restoreAllMocks();
+});
+
+describe('gateway shutdown controller', () => {
+    it('drains resources once when SIGINT and SIGTERM arrive together', async () => {
+        const initialSigint = process.listenerCount('SIGINT');
+        const initialSigterm = process.listenerCount('SIGTERM');
+        let releaseClose = (): void => {};
+        const closeGate = new Promise<void>((resolve) => {
+            releaseClose = resolve;
+        });
+        const close = vi.fn(async () => closeGate);
+        const onStopping = vi.fn();
+        const controller = installGatewayShutdownController({ close, onStopping });
+        controllers.push(controller);
+
+        expect(process.listenerCount('SIGINT')).toBe(initialSigint + 1);
+        expect(process.listenerCount('SIGTERM')).toBe(initialSigterm + 1);
+
+        process.emit('SIGINT');
+        process.emit('SIGTERM');
+        const manualStop = controller.stop('manual');
+        await vi.waitFor(() => expect(close).toHaveBeenCalledTimes(1));
+        releaseClose();
+        await manualStop;
+        await controller.stop('later');
+
+        expect(close).toHaveBeenCalledTimes(1);
+        expect(onStopping).toHaveBeenCalledTimes(1);
+        expect(onStopping).toHaveBeenCalledWith('SIGINT');
+        expect(process.listenerCount('SIGINT')).toBe(initialSigint);
+        expect(process.listenerCount('SIGTERM')).toBe(initialSigterm);
+    });
+
+    it('reports a close failure once and still removes both signal listeners', async () => {
+        const initialSigint = process.listenerCount('SIGINT');
+        const initialSigterm = process.listenerCount('SIGTERM');
+        const failure = new Error('disconnect failed');
+        const onError = vi.fn();
+        const controller = installGatewayShutdownController({
+            close: () => {
+                throw failure;
+            },
+            onError,
+        });
+        controllers.push(controller);
+
+        process.emit('SIGTERM');
+        process.emit('SIGINT');
+        await expect(controller.stop('manual')).rejects.toBe(failure);
+        await vi.waitFor(() => expect(onError).toHaveBeenCalledWith(failure, 'SIGTERM'));
+
+        expect(onError).toHaveBeenCalledTimes(1);
+        expect(process.listenerCount('SIGINT')).toBe(initialSigint);
+        expect(process.listenerCount('SIGTERM')).toBe(initialSigterm);
+    });
+
+    it('can remove unused handlers without closing resources', () => {
+        const close = vi.fn();
+        const controller = installGatewayShutdownController({ close });
+        controllers.push(controller);
+
+        controller.dispose();
+        process.emit('SIGINT');
+
+        expect(close).not.toHaveBeenCalled();
+    });
+});
@@ -44,6 +44,13 @@ DB partial unique index로 한 건만 허용합니다. Turn daemon은 자신의
 Redis 단계가 실패하면 action은 `PARTIAL`과 backoff 상태로 남고 DB 시간은
 다시 이동하지 않습니다.

+Gateway API와 독립 orchestrator의 SIGINT·SIGTERM은
+`installGatewayShutdownController()`가 하나의 종료 Promise로 합칩니다.
+Gateway API는 Fastify `app.close()`를 통해 orchestrator task를 drain한 뒤
+Redis와 PostgreSQL을 닫습니다. 독립 orchestrator도 task drain 뒤 PostgreSQL을
+한 번만 닫습니다. 반복 signal, 종료 완료 뒤 재호출과 close 실패에서도 첫
+reason만 소유하고 등록한 signal listener를 해제합니다.
+
 ## Game API 실행

 `resolveGameApiConfigFromEnv()`가 `PROFILE`, `SCENARIO`,