merge: make gateway shutdown idempotent
This commit is contained in:
@@ -0,0 +1,56 @@
|
|||||||
|
export type GatewayShutdownReason = 'SIGINT' | 'SIGTERM' | string;
|
||||||
|
|
||||||
|
export type GatewayShutdownController = {
|
||||||
|
stop(reason: GatewayShutdownReason): Promise<void>;
|
||||||
|
dispose(): void;
|
||||||
|
};
|
||||||
|
|
||||||
|
type GatewayShutdownOptions = {
|
||||||
|
close(reason: GatewayShutdownReason): void | Promise<void>;
|
||||||
|
onStopping?: (reason: GatewayShutdownReason) => void;
|
||||||
|
onError?: (error: unknown, reason: GatewayShutdownReason) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Installs one idempotent owner for process signals and resource shutdown. */
|
||||||
|
export const installGatewayShutdownController = (options: GatewayShutdownOptions): GatewayShutdownController => {
|
||||||
|
let stopPromise: Promise<void> | undefined;
|
||||||
|
let stopReason: GatewayShutdownReason | undefined;
|
||||||
|
let failureReported = false;
|
||||||
|
let disposed = false;
|
||||||
|
|
||||||
|
const dispose = (): void => {
|
||||||
|
if (disposed) return;
|
||||||
|
disposed = true;
|
||||||
|
process.off('SIGINT', handleSigint);
|
||||||
|
process.off('SIGTERM', handleSigterm);
|
||||||
|
};
|
||||||
|
|
||||||
|
const stop = (reason: GatewayShutdownReason): Promise<void> => {
|
||||||
|
if (stopPromise) return stopPromise;
|
||||||
|
stopReason = reason;
|
||||||
|
options.onStopping?.(reason);
|
||||||
|
stopPromise = Promise.resolve()
|
||||||
|
.then(() => options.close(reason))
|
||||||
|
.finally(dispose);
|
||||||
|
return stopPromise;
|
||||||
|
};
|
||||||
|
|
||||||
|
const requestStop = (reason: GatewayShutdownReason): void => {
|
||||||
|
void stop(reason).catch((error: unknown) => {
|
||||||
|
if (failureReported) return;
|
||||||
|
failureReported = true;
|
||||||
|
options.onError?.(error, stopReason ?? reason);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
function handleSigint(): void {
|
||||||
|
requestStop('SIGINT');
|
||||||
|
}
|
||||||
|
function handleSigterm(): void {
|
||||||
|
requestStop('SIGTERM');
|
||||||
|
}
|
||||||
|
|
||||||
|
process.on('SIGINT', handleSigint);
|
||||||
|
process.on('SIGTERM', handleSigterm);
|
||||||
|
|
||||||
|
return { stop, dispose };
|
||||||
|
};
|
||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
|
|
||||||
import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js';
|
import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js';
|
||||||
import { createGatewayOrchestrator } from './orchestratorFactory.js';
|
import { createGatewayOrchestrator } from './orchestratorFactory.js';
|
||||||
|
import { installGatewayShutdownController } from '../lifecycle/shutdownController.js';
|
||||||
|
|
||||||
export const runGatewayOrchestrator = async (): Promise<void> => {
|
export const runGatewayOrchestrator = async (): Promise<void> => {
|
||||||
const config = resolveGatewayOrchestratorConfigFromEnv();
|
const config = resolveGatewayOrchestratorConfigFromEnv();
|
||||||
@@ -14,15 +15,17 @@ export const runGatewayOrchestrator = async (): Promise<void> => {
|
|||||||
|
|
||||||
const { orchestrator } = createGatewayOrchestrator(postgres.prisma as GatewayPrismaClient, config, process.env);
|
const { orchestrator } = createGatewayOrchestrator(postgres.prisma as GatewayPrismaClient, config, process.env);
|
||||||
|
|
||||||
const stop = async (reason: string): Promise<void> => {
|
|
||||||
console.info(`[gateway-orchestrator] stopping: ${reason}`);
|
|
||||||
await orchestrator.stop();
|
|
||||||
await postgres.disconnect();
|
|
||||||
};
|
|
||||||
|
|
||||||
process.on('SIGINT', () => void stop('SIGINT'));
|
|
||||||
process.on('SIGTERM', () => void stop('SIGTERM'));
|
|
||||||
|
|
||||||
orchestrator.start();
|
orchestrator.start();
|
||||||
|
installGatewayShutdownController({
|
||||||
|
close: async () => {
|
||||||
|
await orchestrator.stop();
|
||||||
|
await postgres.disconnect();
|
||||||
|
},
|
||||||
|
onStopping: (reason) => console.info(`[gateway-orchestrator] stopping: ${reason}`),
|
||||||
|
onError: (error, reason) => {
|
||||||
|
console.error(`[gateway-orchestrator] shutdown failed (${reason})`, error);
|
||||||
|
process.exitCode = 1;
|
||||||
|
},
|
||||||
|
});
|
||||||
console.info('[gateway-orchestrator] started');
|
console.info('[gateway-orchestrator] started');
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import { createGatewayOrchestrator } from './orchestrator/orchestratorFactory.js
|
|||||||
import { appRouter } from './router.js';
|
import { appRouter } from './router.js';
|
||||||
import { RepositoryProfileStatusService } from './lobby/profileStatusService.js';
|
import { RepositoryProfileStatusService } from './lobby/profileStatusService.js';
|
||||||
import { registerAccountIconInternalRoute } from './auth/accountIconInternalRoute.js';
|
import { registerAccountIconInternalRoute } from './auth/accountIconInternalRoute.js';
|
||||||
|
import { installGatewayShutdownController } from './lifecycle/shutdownController.js';
|
||||||
|
|
||||||
export const createGatewayApiServer = async () => {
|
export const createGatewayApiServer = async () => {
|
||||||
const config = resolveGatewayApiConfigFromEnv();
|
const config = resolveGatewayApiConfigFromEnv();
|
||||||
@@ -131,8 +132,23 @@ export const createGatewayApiServer = async () => {
|
|||||||
|
|
||||||
export const runGatewayApiServer = async (): Promise<void> => {
|
export const runGatewayApiServer = async (): Promise<void> => {
|
||||||
const { app, config } = await createGatewayApiServer();
|
const { app, config } = await createGatewayApiServer();
|
||||||
await app.listen({
|
const shutdown = installGatewayShutdownController({
|
||||||
host: config.host,
|
close: () => app.close(),
|
||||||
port: config.port,
|
onStopping: (reason) => app.log.info({ reason }, 'gateway API stopping'),
|
||||||
|
onError: (error, reason) => {
|
||||||
|
app.log.error({ err: error, reason }, 'gateway API shutdown failed');
|
||||||
|
process.exitCode = 1;
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
app.addHook('onClose', async () => shutdown.dispose());
|
||||||
|
try {
|
||||||
|
await app.listen({
|
||||||
|
host: config.host,
|
||||||
|
port: config.port,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
shutdown.dispose();
|
||||||
|
await app.close();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
import { installGatewayShutdownController } from '../src/lifecycle/shutdownController.js';
|
||||||
|
|
||||||
|
const controllers: Array<{ dispose(): void }> = [];
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
for (const controller of controllers.splice(0)) controller.dispose();
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('gateway shutdown controller', () => {
|
||||||
|
it('drains resources once when SIGINT and SIGTERM arrive together', async () => {
|
||||||
|
const initialSigint = process.listenerCount('SIGINT');
|
||||||
|
const initialSigterm = process.listenerCount('SIGTERM');
|
||||||
|
let releaseClose = (): void => {};
|
||||||
|
const closeGate = new Promise<void>((resolve) => {
|
||||||
|
releaseClose = resolve;
|
||||||
|
});
|
||||||
|
const close = vi.fn(async () => closeGate);
|
||||||
|
const onStopping = vi.fn();
|
||||||
|
const controller = installGatewayShutdownController({ close, onStopping });
|
||||||
|
controllers.push(controller);
|
||||||
|
|
||||||
|
expect(process.listenerCount('SIGINT')).toBe(initialSigint + 1);
|
||||||
|
expect(process.listenerCount('SIGTERM')).toBe(initialSigterm + 1);
|
||||||
|
|
||||||
|
process.emit('SIGINT');
|
||||||
|
process.emit('SIGTERM');
|
||||||
|
const manualStop = controller.stop('manual');
|
||||||
|
await vi.waitFor(() => expect(close).toHaveBeenCalledTimes(1));
|
||||||
|
releaseClose();
|
||||||
|
await manualStop;
|
||||||
|
await controller.stop('later');
|
||||||
|
|
||||||
|
expect(close).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onStopping).toHaveBeenCalledTimes(1);
|
||||||
|
expect(onStopping).toHaveBeenCalledWith('SIGINT');
|
||||||
|
expect(process.listenerCount('SIGINT')).toBe(initialSigint);
|
||||||
|
expect(process.listenerCount('SIGTERM')).toBe(initialSigterm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reports a close failure once and still removes both signal listeners', async () => {
|
||||||
|
const initialSigint = process.listenerCount('SIGINT');
|
||||||
|
const initialSigterm = process.listenerCount('SIGTERM');
|
||||||
|
const failure = new Error('disconnect failed');
|
||||||
|
const onError = vi.fn();
|
||||||
|
const controller = installGatewayShutdownController({
|
||||||
|
close: () => {
|
||||||
|
throw failure;
|
||||||
|
},
|
||||||
|
onError,
|
||||||
|
});
|
||||||
|
controllers.push(controller);
|
||||||
|
|
||||||
|
process.emit('SIGTERM');
|
||||||
|
process.emit('SIGINT');
|
||||||
|
await expect(controller.stop('manual')).rejects.toBe(failure);
|
||||||
|
await vi.waitFor(() => expect(onError).toHaveBeenCalledWith(failure, 'SIGTERM'));
|
||||||
|
|
||||||
|
expect(onError).toHaveBeenCalledTimes(1);
|
||||||
|
expect(process.listenerCount('SIGINT')).toBe(initialSigint);
|
||||||
|
expect(process.listenerCount('SIGTERM')).toBe(initialSigterm);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('can remove unused handlers without closing resources', () => {
|
||||||
|
const close = vi.fn();
|
||||||
|
const controller = installGatewayShutdownController({ close });
|
||||||
|
controllers.push(controller);
|
||||||
|
|
||||||
|
controller.dispose();
|
||||||
|
process.emit('SIGINT');
|
||||||
|
|
||||||
|
expect(close).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -44,6 +44,13 @@ DB partial unique index로 한 건만 허용합니다. Turn daemon은 자신의
|
|||||||
Redis 단계가 실패하면 action은 `PARTIAL`과 backoff 상태로 남고 DB 시간은
|
Redis 단계가 실패하면 action은 `PARTIAL`과 backoff 상태로 남고 DB 시간은
|
||||||
다시 이동하지 않습니다.
|
다시 이동하지 않습니다.
|
||||||
|
|
||||||
|
Gateway API와 독립 orchestrator의 SIGINT·SIGTERM은
|
||||||
|
`installGatewayShutdownController()`가 하나의 종료 Promise로 합칩니다.
|
||||||
|
Gateway API는 Fastify `app.close()`를 통해 orchestrator task를 drain한 뒤
|
||||||
|
Redis와 PostgreSQL을 닫습니다. 독립 orchestrator도 task drain 뒤 PostgreSQL을
|
||||||
|
한 번만 닫습니다. 반복 signal, 종료 완료 뒤 재호출과 close 실패에서도 첫
|
||||||
|
reason만 소유하고 등록한 signal listener를 해제합니다.
|
||||||
|
|
||||||
## Game API 실행
|
## Game API 실행
|
||||||
|
|
||||||
`resolveGameApiConfigFromEnv()`가 `PROFILE`, `SCENARIO`,
|
`resolveGameApiConfigFromEnv()`가 `PROFILE`, `SCENARIO`,
|
||||||
|
|||||||
Reference in New Issue
Block a user