fix: Gateway 후속 프로필 큐 정체를 자동 복구

PM2 Node client 세션을 직렬화하고 callback timeout 뒤 다음 orchestrator poll이 계속 진행되게 한다. 운영 판별과 제한 복구 경계 및 회귀 테스트를 함께 기록한다.

app/gateway-api/src/orchestrator/pm2ProcessManager.ts

@@ -8,29 +8,44 @@ import {
     type ProcessDefinition,
 } from './processManager.js';
 
-type Pm2Module = typeof Pm2;
+export interface Pm2Client {
+    connect(callback: (error?: Error) => void): void;
+    disconnect(): void;
+    list(callback: (error: Error | null, list?: Pm2.ProcessDescription[]) => void): void;
+    start(options: Pm2.StartOptions, callback: (error?: Error) => void): void;
+    stop(name: string, callback: (error?: Error) => void): void;
+    delete(name: string, callback: (error?: Error) => void): void;
+}
+
+export interface Pm2ProcessManagerOptions {
+    loadPm2?: () => Pm2Client;
+    connectTimeoutMs?: number;
+    listTimeoutMs?: number;
+    mutationTimeoutMs?: number;
+}
 
 const require = createRequire(import.meta.url);
 
-const loadPm2 = (): Pm2Module => require('pm2') as Pm2Module;
+const loadPm2 = (): Pm2Client => require('pm2') as Pm2Client;
+const DEFAULT_PM2_CONNECT_TIMEOUT_MS = 5_000;
+const DEFAULT_PM2_LIST_TIMEOUT_MS = 5_000;
+const DEFAULT_PM2_MUTATION_TIMEOUT_MS = 30_000;
 
-const withPm2 = async <T>(handler: (pm2: Pm2Module) => Promise<T>): Promise<T> => {
-    const pm2 = loadPm2();
-    await new Promise<void>((resolve, reject) => {
-        pm2.connect((error) => {
-            if (error) {
+const withTimeout = <T>(promise: Promise<T>, timeoutMs: number, label: string): Promise<T> =>
+    new Promise<T>((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms.`)), timeoutMs);
+        timer.unref();
+        promise.then(
+            (value) => {
+                clearTimeout(timer);
+                resolve(value);
+            },
+            (error: unknown) => {
+                clearTimeout(timer);
                 reject(error);
-                return;
             }
-            resolve();
-        });
+        );
     });
-    try {
-        return await handler(pm2);
-    } finally {
-        pm2.disconnect();
-    }
-};
 
 export const buildPm2StartOptions = (definition: ProcessDefinition) => ({
     name: definition.name,
@@ -47,8 +62,52 @@ export const buildPm2StartOptions = (definition: ProcessDefinition) => ({
 });
 
 export class Pm2ProcessManager implements ProcessManager {
+    private readonly loadPm2: () => Pm2Client;
+    private readonly connectTimeoutMs: number;
+    private readonly listTimeoutMs: number;
+    private readonly mutationTimeoutMs: number;
+    private sessionTail: Promise<void> = Promise.resolve();
+
+    constructor(options: Pm2ProcessManagerOptions = {}) {
+        this.loadPm2 = options.loadPm2 ?? loadPm2;
+        this.connectTimeoutMs = options.connectTimeoutMs ?? DEFAULT_PM2_CONNECT_TIMEOUT_MS;
+        this.listTimeoutMs = options.listTimeoutMs ?? DEFAULT_PM2_LIST_TIMEOUT_MS;
+        this.mutationTimeoutMs = options.mutationTimeoutMs ?? DEFAULT_PM2_MUTATION_TIMEOUT_MS;
+    }
+
+    private withPm2<T>(label: string, timeoutMs: number, handler: (pm2: Pm2Client) => Promise<T>): Promise<T> {
+        const task = this.sessionTail.then(async () => {
+            const pm2 = this.loadPm2();
+            try {
+                await withTimeout(
+                    new Promise<void>((resolve, reject) => {
+                        pm2.connect((error) => {
+                            if (error) {
+                                reject(error);
+                                return;
+                            }
+                            resolve();
+                        });
+                    }),
+                    this.connectTimeoutMs,
+                    'PM2 connect'
+                );
+                return await withTimeout(handler(pm2), timeoutMs, label);
+            } finally {
+                pm2.disconnect();
+            }
+        });
+        this.sessionTail = task.then(
+            () => undefined,
+            () => undefined
+        );
+        return task;
+    }
+
     async list(): Promise<ManagedProcessInfo[]> {
-        return withPm2(
+        return this.withPm2(
+            'PM2 list',
+            this.listTimeoutMs,
             (pm2) =>
                 new Promise<ManagedProcessInfo[]>((resolve, reject) => {
                     pm2.list((error, list) => {
@@ -72,7 +131,9 @@ export class Pm2ProcessManager implements ProcessManager {
     }
 
     async start(definition: ProcessDefinition): Promise<void> {
-        await withPm2(
+        await this.withPm2(
+            `PM2 start ${definition.name}`,
+            this.mutationTimeoutMs,
             (pm2) =>
                 new Promise<void>((resolve, reject) => {
                     pm2.list((listError, list) => {
@@ -100,7 +161,9 @@ export class Pm2ProcessManager implements ProcessManager {
     }
 
     async stop(name: string): Promise<void> {
-        await withPm2(
+        await this.withPm2(
+            `PM2 stop ${name}`,
+            this.mutationTimeoutMs,
             (pm2) =>
                 new Promise<void>((resolve, reject) => {
                     pm2.stop(name, (error) => {
@@ -115,7 +178,9 @@ export class Pm2ProcessManager implements ProcessManager {
     }
 
     async delete(name: string): Promise<void> {
-        await withPm2(
+        await this.withPm2(
+            `PM2 delete ${name}`,
+            this.mutationTimeoutMs,
             (pm2) =>
                 new Promise<void>((resolve, reject) => {
                     pm2.delete(name, (error) => {