SSE 연결과 미완료 준비 작업의 자원 상한을 보강한다

2026-10-09 06:30:04 +09:00
1 parent 005f88d7ca
commit c5716cf144
@@ -28,6 +28,10 @@ export class AuthenticatedRealtimeConnection {
        return this.ended;
    }

+    whenIdle(): Promise<void> {
+        return this.queue;
+    }
+
    onClose(cleanup: () => void): void {
        if (this.ended) cleanup();
        else this.cleanup.add(cleanup);
@@ -0,0 +1,93 @@
+const DEFAULT_LIMITS = { connections: 512, perUser: 8, opening: 32 } as const;
+const OPENING_DEADLINE_MS = 5000;
+
+export interface RealtimeConnectionReservation {
+    canContinue(): boolean;
+    reserveUser(userId: string): boolean;
+    startStreaming(): boolean;
+    cancelOpening(): void;
+    release(): void;
+}
+
+/** One game API process owns both pending setup and live streams; no waiting queue. */
+export class RealtimeConnectionAdmission {
+    private readonly limits: { connections: number; perUser: number; opening: number };
+    private total = 0;
+    private opening = 0;
+    private closed = false;
+    private readonly byUser = new Map<string, number>();
+    private readonly pending = new Set<() => void>();
+
+    constructor(limits: Partial<{ connections: number; perUser: number; opening: number }> = {}) {
+        this.limits = { ...DEFAULT_LIMITS, ...limits };
+        if (Object.values(this.limits).some((value) => !Number.isSafeInteger(value) || value < 1))
+            throw new Error('Invalid realtime connection limits.');
+    }
+
+    begin(onTimeout: () => void): RealtimeConnectionReservation | null {
+        if (this.closed || this.total >= this.limits.connections || this.opening >= this.limits.opening) return null;
+        this.total += 1;
+        this.opening += 1;
+        let userId: string | null = null;
+        let streaming = false;
+        let cancelled = false;
+        let released = false;
+        const deadline = Date.now() + OPENING_DEADLINE_MS;
+        const expire = () => {
+            if (released || streaming || cancelled) return;
+            cancelled = true;
+            clearTimeout(timer);
+            onTimeout();
+        };
+        const timer = setTimeout(expire, OPENING_DEADLINE_MS);
+        timer.unref();
+        this.pending.add(expire);
+        const canContinue = () => {
+            if (!released && !streaming && !cancelled && Date.now() >= deadline) expire();
+            return !released && !streaming && !cancelled;
+        };
+        return {
+            canContinue,
+            reserveUser: (id) => {
+                if (!canContinue() || userId !== null) return false;
+                const count = this.byUser.get(id) ?? 0;
+                if (count >= this.limits.perUser) return false;
+                this.byUser.set(id, count + 1);
+                userId = id;
+                return true;
+            },
+            startStreaming: () => {
+                if (!canContinue() || userId === null) return false;
+                streaming = true;
+                this.opening -= 1;
+                clearTimeout(timer);
+                this.pending.delete(expire);
+                return true;
+            },
+            cancelOpening: () => {
+                if (released || streaming) return;
+                cancelled = true;
+                clearTimeout(timer);
+                // Disconnect/timeout does not prove the awaited Redis/SQL operation has settled.
+            },
+            release: () => {
+                if (released) return;
+                released = true;
+                this.total -= 1;
+                if (!streaming) this.opening -= 1;
+                clearTimeout(timer);
+                this.pending.delete(expire);
+                if (userId !== null) {
+                    const count = (this.byUser.get(userId) ?? 1) - 1;
+                    if (count === 0) this.byUser.delete(userId);
+                    else this.byUser.set(userId, count);
+                }
+            },
+        };
+    }
+
+    close(): void {
+        this.closed = true;
+        for (const expire of this.pending) expire();
+    }
+}
@@ -36,6 +36,7 @@ import { RedisBattleSimTransport } from './battleSim/redisTransport.js';
 import { RedisRealtimeEventHub } from './realtime/eventHub.js';
 import { isGameAccessBlocked } from '@sammo-ts/common/auth/sanctions';
 import { AuthenticatedRealtimeConnection } from './realtime/authenticatedConnection.js';
+import { RealtimeConnectionAdmission } from './realtime/connectionAdmission.js';
 import { formatSseFrame } from './realtime/sse.js';
 import {
    shouldForwardRealtimeEvent,
@@ -115,7 +116,9 @@ export const createGameApiServer = async () => {
    });
    app.setErrorHandler(safeHttpErrorHandler);
    const realtimeConnections = new Set<AuthenticatedRealtimeConnection>();
+    const realtimeAdmission = new RealtimeConnectionAdmission();
    app.addHook('preClose', async () => {
+        realtimeAdmission.close();
        for (const connection of realtimeConnections) connection.close();
    });
    const postgres = createGamePostgresConnector(resolvePostgresConfigFromEnv({ schema: config.profile }));
@@ -345,115 +348,165 @@ export const createGameApiServer = async () => {
        const query = request.query as { token?: string; scope?: string };
        const subscriptionScope = query.scope === 'tournament' ? 'tournament' : 'dashboard';
        const tokenFromHeader = extractBearerToken(request.headers.authorization);
-        let auth: GameSessionTokenPayload | null = null;
-        try {
-            auth = query.token === undefined ? await resolveRealtimeAuth(tokenFromHeader) : null;
-        } catch {
-            await reply.status(503).send({ ok: false, error: 'authentication_unavailable' });
-            return;
-        }
-
-        if (!auth || auth.profile !== config.profileName) {
+        if (query.token !== undefined || !tokenFromHeader) {
            await reply.status(401).send({ ok: false, error: 'unauthorized' });
            return;
        }
-
-        const loadViewerIdentity = async (): Promise<RealtimeViewerIdentity> => {
-            const general = await postgres.prisma.general.findFirst({
-                where: { userId: auth.user.id, npcState: 0 },
-                select: { id: true, cityId: true, nationId: true, officerLevel: true, meta: true, penalty: true },
-            });
-            if (!general) {
-                return { generalId: null, cityId: null, nationId: null, canReadDiplomacy: false };
+        let expired = false;
+        let disconnected = false;
+        const opening = realtimeAdmission.begin(() => {
+            expired = true;
+            if (!reply.sent && !reply.raw.destroyed) {
+                reply.header('Retry-After', '15');
+                void reply.status(503).send({ ok: false, error: 'realtime_unavailable' });
            }
-            const nation =
-                general.nationId > 0
-                    ? await postgres.prisma.nation.findUnique({
-                          where: { id: general.nationId },
-                          select: { meta: true },
-                      })
-                    : null;
-            return {
-                generalId: general.id,
-                cityId: general.cityId,
-                nationId: general.nationId,
-                canReadDiplomacy: Boolean(nation && resolveNationPermission(general, nation.meta, false) >= 3),
-            };
-        };
-        let viewerIdentity = await loadViewerIdentity();
-
-        reply.hijack();
-        const requestOrigin = request.headers.origin;
-        if (typeof requestOrigin === 'string' && requestOrigin.length > 0) {
-            // Hijacked SSE responses bypass Fastify's normal CORS response hook.
-            reply.raw.setHeader('Access-Control-Allow-Origin', requestOrigin);
-            reply.raw.setHeader('Access-Control-Allow-Credentials', 'true');
-            reply.raw.setHeader('Vary', 'Origin');
+        });
+        if (!opening) {
+            await reply.header('Retry-After', '15').status(503).send({ ok: false, error: 'realtime_unavailable' });
+            return;
        }
-        reply.raw.setHeader('Content-Type', 'text/event-stream');
-        reply.raw.setHeader('Cache-Control', 'no-cache');
-        reply.raw.setHeader('Connection', 'keep-alive');
-        reply.raw.setHeader('X-Accel-Buffering', 'no');
-        request.raw.setTimeout(0);
-        reply.raw.setTimeout?.(0);
-        reply.raw.flushHeaders?.();
+        const cancelOpening = () => {
+            disconnected = true;
+            opening.cancelOpening();
+        };
+        request.raw.once('aborted', cancelOpening);
+        reply.raw.once('close', cancelOpening);
+        let transferred = false;
+        try {
+            let auth: GameSessionTokenPayload | null = null;
+            try {
+                auth = await resolveRealtimeAuth(tokenFromHeader);
+            } catch {
+                if (opening.canContinue() && !reply.raw.destroyed)
+                    await reply.status(503).send({ ok: false, error: 'authentication_unavailable' });
+                return;
+            }
+            if (!opening.canContinue() || reply.raw.destroyed) return;

-        const connection = new AuthenticatedRealtimeConnection(auth, reply.raw, () =>
-            resolveRealtimeAuth(tokenFromHeader)
-        );
-        realtimeConnections.add(connection);
-        connection.onClose(() => {
-            realtimeConnections.delete(connection);
-        });
-        connection.enqueue(async () => formatSseFrame({ event: 'ready', data: '{}' }));
-        const unsubscribe = realtimeHub.subscribe((event) => {
-            if (!shouldForwardRealtimeEvent(event, subscriptionScope)) return;
-            connection.enqueue(async (currentAuth) => {
-                const identities = [viewerIdentity];
-                if (shouldReloadRealtimeViewerIdentity(event, viewerIdentity)) {
-                    const nextIdentity = await loadViewerIdentity();
-                    identities.push(nextIdentity);
-                    viewerIdentity = nextIdentity;
-                }
-                let refreshGrant: string | undefined;
-                const publicEvent = toPublicRealtimeEvent(event, identities, () => {
-                    refreshGrant ??= createRealtimeAccessGrant(currentAuth, config.profileName, config.gameTokenSecret);
-                    return refreshGrant;
+            if (!auth || auth.profile !== config.profileName) {
+                await reply.status(401).send({ ok: false, error: 'unauthorized' });
+                return;
+            }
+            if (!opening.reserveUser(auth.user.id)) {
+                if (opening.canContinue())
+                    await reply
+                        .header('Retry-After', '15')
+                        .status(429)
+                        .send({ ok: false, error: 'realtime_connection_limit' });
+                return;
+            }
+
+            const loadViewerIdentity = async (): Promise<RealtimeViewerIdentity> => {
+                const general = await postgres.prisma.general.findFirst({
+                    where: { userId: auth.user.id, npcState: 0 },
+                    select: { id: true, cityId: true, nationId: true, officerLevel: true, meta: true, penalty: true },
                });
-                if (!publicEvent || connection.closed) return null;
-                if (refreshGrant) {
-                    try {
-                        await registerRealtimeAccessGrant(redis.client, refreshGrant, config.profileName);
-                    } catch {
-                        // An unregistered grant falls back to the scored refresh path.
+                if (!general) {
+                    return { generalId: null, cityId: null, nationId: null, canReadDiplomacy: false };
+                }
+                const nation =
+                    general.nationId > 0
+                        ? await postgres.prisma.nation.findUnique({
+                              where: { id: general.nationId },
+                              select: { meta: true },
+                          })
+                        : null;
+                return {
+                    generalId: general.id,
+                    cityId: general.cityId,
+                    nationId: general.nationId,
+                    canReadDiplomacy: Boolean(nation && resolveNationPermission(general, nation.meta, false) >= 3),
+                };
+            };
+            let viewerIdentity = await loadViewerIdentity();
+            if (!opening.canContinue() || reply.raw.destroyed || !opening.startStreaming()) return;
+
+            reply.hijack();
+            const requestOrigin = request.headers.origin;
+            if (typeof requestOrigin === 'string' && requestOrigin.length > 0) {
+                // Hijacked SSE responses bypass Fastify's normal CORS response hook.
+                reply.raw.setHeader('Access-Control-Allow-Origin', requestOrigin);
+                reply.raw.setHeader('Access-Control-Allow-Credentials', 'true');
+                reply.raw.setHeader('Vary', 'Origin');
+            }
+            reply.raw.setHeader('Content-Type', 'text/event-stream');
+            reply.raw.setHeader('Cache-Control', 'no-cache');
+            reply.raw.setHeader('Connection', 'keep-alive');
+            reply.raw.setHeader('X-Accel-Buffering', 'no');
+            request.raw.setTimeout(0);
+            reply.raw.setTimeout?.(0);
+            reply.raw.flushHeaders?.();
+
+            const connection = new AuthenticatedRealtimeConnection(auth, reply.raw, () =>
+                resolveRealtimeAuth(tokenFromHeader)
+            );
+            realtimeConnections.add(connection);
+            connection.onClose(() => {
+                realtimeConnections.delete(connection);
+                // A disconnected socket can still own an in-flight Redis/SQL operation.
+                void connection.whenIdle().then(opening.release, opening.release);
+            });
+            transferred = true;
+            connection.enqueue(async () => formatSseFrame({ event: 'ready', data: '{}' }));
+            const unsubscribe = realtimeHub.subscribe((event) => {
+                if (!shouldForwardRealtimeEvent(event, subscriptionScope)) return;
+                connection.enqueue(async (currentAuth) => {
+                    const identities = [viewerIdentity];
+                    if (shouldReloadRealtimeViewerIdentity(event, viewerIdentity)) {
+                        const nextIdentity = await loadViewerIdentity();
+                        identities.push(nextIdentity);
+                        viewerIdentity = nextIdentity;
                    }
-                }
-                return formatSseFrame({ event: publicEvent.type, data: JSON.stringify(publicEvent) });
-            });
-        });
-        connection.onClose(unsubscribe);
-        let heartbeatPending = false;
-        const heartbeat = setInterval(() => {
-            if (heartbeatPending || connection.closed) return;
-            heartbeatPending = true;
-            connection.enqueue(async () => {
-                try {
-                    return formatSseFrame({
-                        event: 'ping',
-                        data: JSON.stringify({ turnEngineRunning: await turnEngineStatus.get() }),
+                    let refreshGrant: string | undefined;
+                    const publicEvent = toPublicRealtimeEvent(event, identities, () => {
+                        refreshGrant ??= createRealtimeAccessGrant(
+                            currentAuth,
+                            config.profileName,
+                            config.gameTokenSecret
+                        );
+                        return refreshGrant;
                    });
-                } finally {
-                    heartbeatPending = false;
-                }
+                    if (!publicEvent || connection.closed) return null;
+                    if (refreshGrant) {
+                        try {
+                            await registerRealtimeAccessGrant(redis.client, refreshGrant, config.profileName);
+                        } catch {
+                            // An unregistered grant falls back to the scored refresh path.
+                        }
+                    }
+                    return formatSseFrame({ event: publicEvent.type, data: JSON.stringify(publicEvent) });
+                });
            });
-        }, 15_000);
-        connection.onClose(() => {
-            clearInterval(heartbeat);
-        });
-        request.raw.once('aborted', connection.close);
-        connection.onClose(() => {
-            request.raw.off('aborted', connection.close);
-        });
+            connection.onClose(unsubscribe);
+            let heartbeatPending = false;
+            const heartbeat = setInterval(() => {
+                if (heartbeatPending || connection.closed) return;
+                heartbeatPending = true;
+                connection.enqueue(async () => {
+                    try {
+                        return formatSseFrame({
+                            event: 'ping',
+                            data: JSON.stringify({ turnEngineRunning: await turnEngineStatus.get() }),
+                        });
+                    } finally {
+                        heartbeatPending = false;
+                    }
+                });
+            }, 15_000);
+            connection.onClose(() => {
+                clearInterval(heartbeat);
+            });
+            request.raw.once('aborted', connection.close);
+            connection.onClose(() => {
+                request.raw.off('aborted', connection.close);
+            });
+        } catch (error) {
+            if (!expired && !disconnected) throw error;
+        } finally {
+            request.raw.off('aborted', cancelOpening);
+            reply.raw.off('close', cancelOpening);
+            if (!transferred) opening.release();
+        }
    });

    app.get('/healthz', async (_request, reply) => {