SSE 연결과 미완료 준비 작업의 자원 상한을 보강한다
This commit is contained in:
@@ -28,6 +28,10 @@ export class AuthenticatedRealtimeConnection {
|
||||
return this.ended;
|
||||
}
|
||||
|
||||
whenIdle(): Promise<void> {
|
||||
return this.queue;
|
||||
}
|
||||
|
||||
onClose(cleanup: () => void): void {
|
||||
if (this.ended) cleanup();
|
||||
else this.cleanup.add(cleanup);
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
const DEFAULT_LIMITS = { connections: 512, perUser: 8, opening: 32 } as const;
|
||||
const OPENING_DEADLINE_MS = 5000;
|
||||
|
||||
export interface RealtimeConnectionReservation {
|
||||
canContinue(): boolean;
|
||||
reserveUser(userId: string): boolean;
|
||||
startStreaming(): boolean;
|
||||
cancelOpening(): void;
|
||||
release(): void;
|
||||
}
|
||||
|
||||
/** One game API process owns both pending setup and live streams; no waiting queue. */
|
||||
export class RealtimeConnectionAdmission {
|
||||
private readonly limits: { connections: number; perUser: number; opening: number };
|
||||
private total = 0;
|
||||
private opening = 0;
|
||||
private closed = false;
|
||||
private readonly byUser = new Map<string, number>();
|
||||
private readonly pending = new Set<() => void>();
|
||||
|
||||
constructor(limits: Partial<{ connections: number; perUser: number; opening: number }> = {}) {
|
||||
this.limits = { ...DEFAULT_LIMITS, ...limits };
|
||||
if (Object.values(this.limits).some((value) => !Number.isSafeInteger(value) || value < 1))
|
||||
throw new Error('Invalid realtime connection limits.');
|
||||
}
|
||||
|
||||
begin(onTimeout: () => void): RealtimeConnectionReservation | null {
|
||||
if (this.closed || this.total >= this.limits.connections || this.opening >= this.limits.opening) return null;
|
||||
this.total += 1;
|
||||
this.opening += 1;
|
||||
let userId: string | null = null;
|
||||
let streaming = false;
|
||||
let cancelled = false;
|
||||
let released = false;
|
||||
const deadline = Date.now() + OPENING_DEADLINE_MS;
|
||||
const expire = () => {
|
||||
if (released || streaming || cancelled) return;
|
||||
cancelled = true;
|
||||
clearTimeout(timer);
|
||||
onTimeout();
|
||||
};
|
||||
const timer = setTimeout(expire, OPENING_DEADLINE_MS);
|
||||
timer.unref();
|
||||
this.pending.add(expire);
|
||||
const canContinue = () => {
|
||||
if (!released && !streaming && !cancelled && Date.now() >= deadline) expire();
|
||||
return !released && !streaming && !cancelled;
|
||||
};
|
||||
return {
|
||||
canContinue,
|
||||
reserveUser: (id) => {
|
||||
if (!canContinue() || userId !== null) return false;
|
||||
const count = this.byUser.get(id) ?? 0;
|
||||
if (count >= this.limits.perUser) return false;
|
||||
this.byUser.set(id, count + 1);
|
||||
userId = id;
|
||||
return true;
|
||||
},
|
||||
startStreaming: () => {
|
||||
if (!canContinue() || userId === null) return false;
|
||||
streaming = true;
|
||||
this.opening -= 1;
|
||||
clearTimeout(timer);
|
||||
this.pending.delete(expire);
|
||||
return true;
|
||||
},
|
||||
cancelOpening: () => {
|
||||
if (released || streaming) return;
|
||||
cancelled = true;
|
||||
clearTimeout(timer);
|
||||
// Disconnect/timeout does not prove the awaited Redis/SQL operation has settled.
|
||||
},
|
||||
release: () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
this.total -= 1;
|
||||
if (!streaming) this.opening -= 1;
|
||||
clearTimeout(timer);
|
||||
this.pending.delete(expire);
|
||||
if (userId !== null) {
|
||||
const count = (this.byUser.get(userId) ?? 1) - 1;
|
||||
if (count === 0) this.byUser.delete(userId);
|
||||
else this.byUser.set(userId, count);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.closed = true;
|
||||
for (const expire of this.pending) expire();
|
||||
}
|
||||
}
|
||||
+152
-99
@@ -36,6 +36,7 @@ import { RedisBattleSimTransport } from './battleSim/redisTransport.js';
|
||||
import { RedisRealtimeEventHub } from './realtime/eventHub.js';
|
||||
import { isGameAccessBlocked } from '@sammo-ts/common/auth/sanctions';
|
||||
import { AuthenticatedRealtimeConnection } from './realtime/authenticatedConnection.js';
|
||||
import { RealtimeConnectionAdmission } from './realtime/connectionAdmission.js';
|
||||
import { formatSseFrame } from './realtime/sse.js';
|
||||
import {
|
||||
shouldForwardRealtimeEvent,
|
||||
@@ -115,7 +116,9 @@ export const createGameApiServer = async () => {
|
||||
});
|
||||
app.setErrorHandler(safeHttpErrorHandler);
|
||||
const realtimeConnections = new Set<AuthenticatedRealtimeConnection>();
|
||||
const realtimeAdmission = new RealtimeConnectionAdmission();
|
||||
app.addHook('preClose', async () => {
|
||||
realtimeAdmission.close();
|
||||
for (const connection of realtimeConnections) connection.close();
|
||||
});
|
||||
const postgres = createGamePostgresConnector(resolvePostgresConfigFromEnv({ schema: config.profile }));
|
||||
@@ -345,115 +348,165 @@ export const createGameApiServer = async () => {
|
||||
const query = request.query as { token?: string; scope?: string };
|
||||
const subscriptionScope = query.scope === 'tournament' ? 'tournament' : 'dashboard';
|
||||
const tokenFromHeader = extractBearerToken(request.headers.authorization);
|
||||
let auth: GameSessionTokenPayload | null = null;
|
||||
try {
|
||||
auth = query.token === undefined ? await resolveRealtimeAuth(tokenFromHeader) : null;
|
||||
} catch {
|
||||
await reply.status(503).send({ ok: false, error: 'authentication_unavailable' });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!auth || auth.profile !== config.profileName) {
|
||||
if (query.token !== undefined || !tokenFromHeader) {
|
||||
await reply.status(401).send({ ok: false, error: 'unauthorized' });
|
||||
return;
|
||||
}
|
||||
|
||||
const loadViewerIdentity = async (): Promise<RealtimeViewerIdentity> => {
|
||||
const general = await postgres.prisma.general.findFirst({
|
||||
where: { userId: auth.user.id, npcState: 0 },
|
||||
select: { id: true, cityId: true, nationId: true, officerLevel: true, meta: true, penalty: true },
|
||||
});
|
||||
if (!general) {
|
||||
return { generalId: null, cityId: null, nationId: null, canReadDiplomacy: false };
|
||||
let expired = false;
|
||||
let disconnected = false;
|
||||
const opening = realtimeAdmission.begin(() => {
|
||||
expired = true;
|
||||
if (!reply.sent && !reply.raw.destroyed) {
|
||||
reply.header('Retry-After', '15');
|
||||
void reply.status(503).send({ ok: false, error: 'realtime_unavailable' });
|
||||
}
|
||||
const nation =
|
||||
general.nationId > 0
|
||||
? await postgres.prisma.nation.findUnique({
|
||||
where: { id: general.nationId },
|
||||
select: { meta: true },
|
||||
})
|
||||
: null;
|
||||
return {
|
||||
generalId: general.id,
|
||||
cityId: general.cityId,
|
||||
nationId: general.nationId,
|
||||
canReadDiplomacy: Boolean(nation && resolveNationPermission(general, nation.meta, false) >= 3),
|
||||
};
|
||||
};
|
||||
let viewerIdentity = await loadViewerIdentity();
|
||||
|
||||
reply.hijack();
|
||||
const requestOrigin = request.headers.origin;
|
||||
if (typeof requestOrigin === 'string' && requestOrigin.length > 0) {
|
||||
// Hijacked SSE responses bypass Fastify's normal CORS response hook.
|
||||
reply.raw.setHeader('Access-Control-Allow-Origin', requestOrigin);
|
||||
reply.raw.setHeader('Access-Control-Allow-Credentials', 'true');
|
||||
reply.raw.setHeader('Vary', 'Origin');
|
||||
});
|
||||
if (!opening) {
|
||||
await reply.header('Retry-After', '15').status(503).send({ ok: false, error: 'realtime_unavailable' });
|
||||
return;
|
||||
}
|
||||
reply.raw.setHeader('Content-Type', 'text/event-stream');
|
||||
reply.raw.setHeader('Cache-Control', 'no-cache');
|
||||
reply.raw.setHeader('Connection', 'keep-alive');
|
||||
reply.raw.setHeader('X-Accel-Buffering', 'no');
|
||||
request.raw.setTimeout(0);
|
||||
reply.raw.setTimeout?.(0);
|
||||
reply.raw.flushHeaders?.();
|
||||
const cancelOpening = () => {
|
||||
disconnected = true;
|
||||
opening.cancelOpening();
|
||||
};
|
||||
request.raw.once('aborted', cancelOpening);
|
||||
reply.raw.once('close', cancelOpening);
|
||||
let transferred = false;
|
||||
try {
|
||||
let auth: GameSessionTokenPayload | null = null;
|
||||
try {
|
||||
auth = await resolveRealtimeAuth(tokenFromHeader);
|
||||
} catch {
|
||||
if (opening.canContinue() && !reply.raw.destroyed)
|
||||
await reply.status(503).send({ ok: false, error: 'authentication_unavailable' });
|
||||
return;
|
||||
}
|
||||
if (!opening.canContinue() || reply.raw.destroyed) return;
|
||||
|
||||
const connection = new AuthenticatedRealtimeConnection(auth, reply.raw, () =>
|
||||
resolveRealtimeAuth(tokenFromHeader)
|
||||
);
|
||||
realtimeConnections.add(connection);
|
||||
connection.onClose(() => {
|
||||
realtimeConnections.delete(connection);
|
||||
});
|
||||
connection.enqueue(async () => formatSseFrame({ event: 'ready', data: '{}' }));
|
||||
const unsubscribe = realtimeHub.subscribe((event) => {
|
||||
if (!shouldForwardRealtimeEvent(event, subscriptionScope)) return;
|
||||
connection.enqueue(async (currentAuth) => {
|
||||
const identities = [viewerIdentity];
|
||||
if (shouldReloadRealtimeViewerIdentity(event, viewerIdentity)) {
|
||||
const nextIdentity = await loadViewerIdentity();
|
||||
identities.push(nextIdentity);
|
||||
viewerIdentity = nextIdentity;
|
||||
}
|
||||
let refreshGrant: string | undefined;
|
||||
const publicEvent = toPublicRealtimeEvent(event, identities, () => {
|
||||
refreshGrant ??= createRealtimeAccessGrant(currentAuth, config.profileName, config.gameTokenSecret);
|
||||
return refreshGrant;
|
||||
if (!auth || auth.profile !== config.profileName) {
|
||||
await reply.status(401).send({ ok: false, error: 'unauthorized' });
|
||||
return;
|
||||
}
|
||||
if (!opening.reserveUser(auth.user.id)) {
|
||||
if (opening.canContinue())
|
||||
await reply
|
||||
.header('Retry-After', '15')
|
||||
.status(429)
|
||||
.send({ ok: false, error: 'realtime_connection_limit' });
|
||||
return;
|
||||
}
|
||||
|
||||
const loadViewerIdentity = async (): Promise<RealtimeViewerIdentity> => {
|
||||
const general = await postgres.prisma.general.findFirst({
|
||||
where: { userId: auth.user.id, npcState: 0 },
|
||||
select: { id: true, cityId: true, nationId: true, officerLevel: true, meta: true, penalty: true },
|
||||
});
|
||||
if (!publicEvent || connection.closed) return null;
|
||||
if (refreshGrant) {
|
||||
try {
|
||||
await registerRealtimeAccessGrant(redis.client, refreshGrant, config.profileName);
|
||||
} catch {
|
||||
// An unregistered grant falls back to the scored refresh path.
|
||||
if (!general) {
|
||||
return { generalId: null, cityId: null, nationId: null, canReadDiplomacy: false };
|
||||
}
|
||||
const nation =
|
||||
general.nationId > 0
|
||||
? await postgres.prisma.nation.findUnique({
|
||||
where: { id: general.nationId },
|
||||
select: { meta: true },
|
||||
})
|
||||
: null;
|
||||
return {
|
||||
generalId: general.id,
|
||||
cityId: general.cityId,
|
||||
nationId: general.nationId,
|
||||
canReadDiplomacy: Boolean(nation && resolveNationPermission(general, nation.meta, false) >= 3),
|
||||
};
|
||||
};
|
||||
let viewerIdentity = await loadViewerIdentity();
|
||||
if (!opening.canContinue() || reply.raw.destroyed || !opening.startStreaming()) return;
|
||||
|
||||
reply.hijack();
|
||||
const requestOrigin = request.headers.origin;
|
||||
if (typeof requestOrigin === 'string' && requestOrigin.length > 0) {
|
||||
// Hijacked SSE responses bypass Fastify's normal CORS response hook.
|
||||
reply.raw.setHeader('Access-Control-Allow-Origin', requestOrigin);
|
||||
reply.raw.setHeader('Access-Control-Allow-Credentials', 'true');
|
||||
reply.raw.setHeader('Vary', 'Origin');
|
||||
}
|
||||
reply.raw.setHeader('Content-Type', 'text/event-stream');
|
||||
reply.raw.setHeader('Cache-Control', 'no-cache');
|
||||
reply.raw.setHeader('Connection', 'keep-alive');
|
||||
reply.raw.setHeader('X-Accel-Buffering', 'no');
|
||||
request.raw.setTimeout(0);
|
||||
reply.raw.setTimeout?.(0);
|
||||
reply.raw.flushHeaders?.();
|
||||
|
||||
const connection = new AuthenticatedRealtimeConnection(auth, reply.raw, () =>
|
||||
resolveRealtimeAuth(tokenFromHeader)
|
||||
);
|
||||
realtimeConnections.add(connection);
|
||||
connection.onClose(() => {
|
||||
realtimeConnections.delete(connection);
|
||||
// A disconnected socket can still own an in-flight Redis/SQL operation.
|
||||
void connection.whenIdle().then(opening.release, opening.release);
|
||||
});
|
||||
transferred = true;
|
||||
connection.enqueue(async () => formatSseFrame({ event: 'ready', data: '{}' }));
|
||||
const unsubscribe = realtimeHub.subscribe((event) => {
|
||||
if (!shouldForwardRealtimeEvent(event, subscriptionScope)) return;
|
||||
connection.enqueue(async (currentAuth) => {
|
||||
const identities = [viewerIdentity];
|
||||
if (shouldReloadRealtimeViewerIdentity(event, viewerIdentity)) {
|
||||
const nextIdentity = await loadViewerIdentity();
|
||||
identities.push(nextIdentity);
|
||||
viewerIdentity = nextIdentity;
|
||||
}
|
||||
}
|
||||
return formatSseFrame({ event: publicEvent.type, data: JSON.stringify(publicEvent) });
|
||||
});
|
||||
});
|
||||
connection.onClose(unsubscribe);
|
||||
let heartbeatPending = false;
|
||||
const heartbeat = setInterval(() => {
|
||||
if (heartbeatPending || connection.closed) return;
|
||||
heartbeatPending = true;
|
||||
connection.enqueue(async () => {
|
||||
try {
|
||||
return formatSseFrame({
|
||||
event: 'ping',
|
||||
data: JSON.stringify({ turnEngineRunning: await turnEngineStatus.get() }),
|
||||
let refreshGrant: string | undefined;
|
||||
const publicEvent = toPublicRealtimeEvent(event, identities, () => {
|
||||
refreshGrant ??= createRealtimeAccessGrant(
|
||||
currentAuth,
|
||||
config.profileName,
|
||||
config.gameTokenSecret
|
||||
);
|
||||
return refreshGrant;
|
||||
});
|
||||
} finally {
|
||||
heartbeatPending = false;
|
||||
}
|
||||
if (!publicEvent || connection.closed) return null;
|
||||
if (refreshGrant) {
|
||||
try {
|
||||
await registerRealtimeAccessGrant(redis.client, refreshGrant, config.profileName);
|
||||
} catch {
|
||||
// An unregistered grant falls back to the scored refresh path.
|
||||
}
|
||||
}
|
||||
return formatSseFrame({ event: publicEvent.type, data: JSON.stringify(publicEvent) });
|
||||
});
|
||||
});
|
||||
}, 15_000);
|
||||
connection.onClose(() => {
|
||||
clearInterval(heartbeat);
|
||||
});
|
||||
request.raw.once('aborted', connection.close);
|
||||
connection.onClose(() => {
|
||||
request.raw.off('aborted', connection.close);
|
||||
});
|
||||
connection.onClose(unsubscribe);
|
||||
let heartbeatPending = false;
|
||||
const heartbeat = setInterval(() => {
|
||||
if (heartbeatPending || connection.closed) return;
|
||||
heartbeatPending = true;
|
||||
connection.enqueue(async () => {
|
||||
try {
|
||||
return formatSseFrame({
|
||||
event: 'ping',
|
||||
data: JSON.stringify({ turnEngineRunning: await turnEngineStatus.get() }),
|
||||
});
|
||||
} finally {
|
||||
heartbeatPending = false;
|
||||
}
|
||||
});
|
||||
}, 15_000);
|
||||
connection.onClose(() => {
|
||||
clearInterval(heartbeat);
|
||||
});
|
||||
request.raw.once('aborted', connection.close);
|
||||
connection.onClose(() => {
|
||||
request.raw.off('aborted', connection.close);
|
||||
});
|
||||
} catch (error) {
|
||||
if (!expired && !disconnected) throw error;
|
||||
} finally {
|
||||
request.raw.off('aborted', cancelOpening);
|
||||
reply.raw.off('close', cancelOpening);
|
||||
if (!transferred) opening.release();
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/healthz', async (_request, reply) => {
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { PassThrough } from 'node:stream';
|
||||
import type { ServerResponse } from 'node:http';
|
||||
import type { GameSessionTokenPayload } from '@sammo-ts/common/auth/gameToken';
|
||||
import { RealtimeConnectionAdmission } from '../src/realtime/connectionAdmission.js';
|
||||
import { AuthenticatedRealtimeConnection } from '../src/realtime/authenticatedConnection.js';
|
||||
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
describe('realtime connection admission', () => {
|
||||
it('reserves the process total before authentication and counts live streams too', () => {
|
||||
const budget = new RealtimeConnectionAdmission({ connections: 2, perUser: 2, opening: 2 });
|
||||
const first = budget.begin(vi.fn())!;
|
||||
const second = budget.begin(vi.fn())!;
|
||||
expect(first.reserveUser('actor')).toBe(true);
|
||||
expect(first.startStreaming()).toBe(true);
|
||||
expect(budget.begin(vi.fn())).toBeNull();
|
||||
second.release();
|
||||
const replacement = budget.begin(vi.fn())!;
|
||||
expect(replacement).not.toBeNull();
|
||||
replacement.release();
|
||||
first.release();
|
||||
});
|
||||
|
||||
it('shares the authenticated actor limit while allowing another actor', () => {
|
||||
const budget = new RealtimeConnectionAdmission({ connections: 5, perUser: 1, opening: 5 });
|
||||
const first = budget.begin(vi.fn())!;
|
||||
expect(first.reserveUser('one')).toBe(true);
|
||||
const rejected = budget.begin(vi.fn())!;
|
||||
expect(rejected.reserveUser('one')).toBe(false);
|
||||
expect(rejected.startStreaming()).toBe(false);
|
||||
rejected.release();
|
||||
const other = budget.begin(vi.fn())!;
|
||||
expect(other.reserveUser('two')).toBe(true);
|
||||
first.release();
|
||||
first.release();
|
||||
const replacement = budget.begin(vi.fn())!;
|
||||
expect(replacement.reserveUser('one')).toBe(true);
|
||||
other.release();
|
||||
replacement.release();
|
||||
});
|
||||
|
||||
it('keeps timed-out and disconnected backend work admitted until it actually settles', async () => {
|
||||
vi.useFakeTimers();
|
||||
const budget = new RealtimeConnectionAdmission({ connections: 5, perUser: 5, opening: 2 });
|
||||
const timeout = vi.fn();
|
||||
const stalled = budget.begin(timeout)!;
|
||||
const disconnected = budget.begin(vi.fn())!;
|
||||
expect(stalled.reserveUser('stalled')).toBe(true);
|
||||
disconnected.cancelOpening();
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(timeout).toHaveBeenCalledTimes(1);
|
||||
expect(stalled.canContinue()).toBe(false);
|
||||
expect(stalled.startStreaming()).toBe(false);
|
||||
expect(disconnected.canContinue()).toBe(false);
|
||||
expect(budget.begin(vi.fn())).toBeNull();
|
||||
stalled.release();
|
||||
const recovered = budget.begin(vi.fn())!;
|
||||
expect(recovered.reserveUser('stalled')).toBe(true);
|
||||
expect(recovered.startStreaming()).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(timeout).toHaveBeenCalledTimes(1);
|
||||
disconnected.release();
|
||||
recovered.release();
|
||||
});
|
||||
|
||||
it('stops pending setup on shutdown and permanently refuses new admissions', () => {
|
||||
const budget = new RealtimeConnectionAdmission();
|
||||
const timeout = vi.fn();
|
||||
const pending = budget.begin(timeout)!;
|
||||
budget.close();
|
||||
budget.close();
|
||||
expect(timeout).toHaveBeenCalledTimes(1);
|
||||
expect(pending.canContinue()).toBe(false);
|
||||
expect(budget.begin(vi.fn())).toBeNull();
|
||||
pending.release();
|
||||
});
|
||||
|
||||
it('keeps a closed stream counted while its in-flight authentication is still pending', async () => {
|
||||
const budget = new RealtimeConnectionAdmission({ connections: 1, perUser: 1, opening: 1 });
|
||||
const slot = budget.begin(vi.fn())!;
|
||||
expect(slot.reserveUser('actor')).toBe(true);
|
||||
expect(slot.startStreaming()).toBe(true);
|
||||
let finish!: () => void;
|
||||
let started!: () => void;
|
||||
const inFlight = new Promise<void>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
const running = new Promise<void>((resolve) => {
|
||||
started = resolve;
|
||||
});
|
||||
const auth: GameSessionTokenPayload = {
|
||||
version: 1,
|
||||
sessionId: 'session',
|
||||
profile: 'che:default',
|
||||
issuedAt: new Date().toISOString(),
|
||||
expiresAt: new Date(Date.now() + 60_000).toISOString(),
|
||||
user: {
|
||||
id: 'actor',
|
||||
username: 'actor',
|
||||
displayName: 'actor',
|
||||
roles: ['user'],
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
sanctions: {},
|
||||
};
|
||||
const response = new PassThrough();
|
||||
const connection = new AuthenticatedRealtimeConnection(
|
||||
auth,
|
||||
response as unknown as ServerResponse,
|
||||
async () => {
|
||||
started();
|
||||
await inFlight;
|
||||
return auth;
|
||||
}
|
||||
);
|
||||
connection.onClose(() => {
|
||||
void connection.whenIdle().then(slot.release, slot.release);
|
||||
});
|
||||
connection.enqueue(async () => 'event: ready\ndata: {}\n\n');
|
||||
await running;
|
||||
connection.close();
|
||||
expect(budget.begin(vi.fn())).toBeNull();
|
||||
finish();
|
||||
await connection.whenIdle();
|
||||
const replacement = budget.begin(vi.fn())!;
|
||||
expect(replacement.reserveUser('actor')).toBe(true);
|
||||
replacement.release();
|
||||
});
|
||||
|
||||
it('enforces the production 512 total and 32 pending openings without accumulating a queue', () => {
|
||||
const budget = new RealtimeConnectionAdmission();
|
||||
const pending = Array.from({ length: 32 }, () => budget.begin(vi.fn())!);
|
||||
expect(pending.every(Boolean)).toBe(true);
|
||||
expect(budget.begin(vi.fn())).toBeNull();
|
||||
for (const slot of pending) slot.release();
|
||||
const live = Array.from({ length: 512 }, (_, index) => {
|
||||
const slot = budget.begin(vi.fn())!;
|
||||
expect(slot.reserveUser(`actor-${index}`)).toBe(true);
|
||||
expect(slot.startStreaming()).toBe(true);
|
||||
return slot;
|
||||
});
|
||||
expect(budget.begin(vi.fn())).toBeNull();
|
||||
live[0]!.release();
|
||||
const replacement = budget.begin(vi.fn())!;
|
||||
expect(replacement.reserveUser('replacement')).toBe(true);
|
||||
expect(replacement.startStreaming()).toBe(true);
|
||||
for (const slot of live) slot.release();
|
||||
replacement.release();
|
||||
});
|
||||
});
|
||||
@@ -3589,6 +3589,168 @@ integration('game API security over HTTP transport', () => {
|
||||
return { abort, reader };
|
||||
};
|
||||
|
||||
it('SSE limits simultaneous openings for one authenticated user', async () => {
|
||||
const actor = `security-http-sse-concurrent-${process.pid}`;
|
||||
const token = await createAccessToken('sse-concurrent', {}, actor);
|
||||
const aborts = Array.from({ length: 10 }, () => new AbortController());
|
||||
try {
|
||||
const responses = await Promise.all(
|
||||
aborts.map((abort) =>
|
||||
fetch(`${baseUrl}${server!.config.eventsPath}`, {
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
signal: abort.signal,
|
||||
})
|
||||
)
|
||||
);
|
||||
expect(responses.filter((response) => response.status === 200)).toHaveLength(8);
|
||||
expect(responses.filter((response) => response.status === 429)).toHaveLength(2);
|
||||
for (const response of responses.filter((item) => item.status === 429)) {
|
||||
expect(response.headers.get('retry-after')).toBe('15');
|
||||
expect(response.headers.get('content-type')).not.toBe('text/event-stream');
|
||||
expect(await response.json()).toEqual({ ok: false, error: 'realtime_connection_limit' });
|
||||
}
|
||||
} finally {
|
||||
for (const abort of aborts) abort.abort();
|
||||
await accessTokenStore.revoke(token);
|
||||
}
|
||||
});
|
||||
|
||||
it('SSE limits the same user across tokens and scopes and recovers after disconnect', async () => {
|
||||
const actor = `security-http-sse-shared-${process.pid}`;
|
||||
const token = await createAccessToken('sse-shared-a', {}, actor);
|
||||
const otherToken = await createAccessToken('sse-shared-b', {}, actor);
|
||||
const otherActorToken = await createAccessToken('sse-independent', {}, `${actor}-other`);
|
||||
const streams: Awaited<ReturnType<typeof openRealtime>>[] = [];
|
||||
const replacementAbort = new AbortController();
|
||||
try {
|
||||
for (let index = 0; index < 8; index += 1) streams.push(await openRealtime(token));
|
||||
const rejected = await fetch(`${baseUrl}${server!.config.eventsPath}?scope=tournament`, {
|
||||
headers: { authorization: `Bearer ${otherToken}` },
|
||||
});
|
||||
expect(rejected.status).toBe(429);
|
||||
expect(await rejected.json()).toEqual({ ok: false, error: 'realtime_connection_limit' });
|
||||
streams.push(await openRealtime(otherActorToken));
|
||||
await streams[0]!.reader.cancel();
|
||||
streams[0]!.abort.abort();
|
||||
let replacement: Response | undefined;
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const response = await fetch(`${baseUrl}${server!.config.eventsPath}?scope=tournament`, {
|
||||
headers: { authorization: `Bearer ${otherToken}` },
|
||||
signal: replacementAbort.signal,
|
||||
});
|
||||
if (response.status === 200) replacement = response;
|
||||
else await response.arrayBuffer();
|
||||
return response.status;
|
||||
})
|
||||
.toBe(200);
|
||||
const ready = await replacement!.body!.getReader().read();
|
||||
expect(new TextDecoder().decode(ready.value)).toContain('event: ready');
|
||||
} finally {
|
||||
replacementAbort.abort();
|
||||
for (const stream of streams) stream.abort.abort();
|
||||
await Promise.all([token, otherToken, otherActorToken].map((value) => accessTokenStore.revoke(value)));
|
||||
}
|
||||
});
|
||||
|
||||
it('SSE bounds pending openings after disconnect and backend deadline until real SQL work settles', async () => {
|
||||
if (!/^[A-Za-z0-9_]+$/.test(profileId)) throw new Error('Invalid isolated schema identifier');
|
||||
const actor = `security-http-sse-pending-${process.pid}`;
|
||||
const tokens = await Promise.all(
|
||||
Array.from({ length: 33 }, (_, index) => createAccessToken(`sse-pending-${index}`, {}, `${actor}-${index}`))
|
||||
);
|
||||
const aborts = Array.from({ length: 33 }, () => new AbortController());
|
||||
let unlock!: () => void;
|
||||
const release = new Promise<void>((resolve) => {
|
||||
unlock = resolve;
|
||||
});
|
||||
let ready!: (tx: GamePrisma.TransactionClient) => void;
|
||||
let lockError!: (error: unknown) => void;
|
||||
const locked = new Promise<GamePrisma.TransactionClient>((resolve, reject) => {
|
||||
ready = resolve;
|
||||
lockError = reject;
|
||||
});
|
||||
// Only the guarded, dedicated fixture schema is locked. No runtime or Ref table is touched.
|
||||
const locking = db.$transaction(
|
||||
async (tx) => {
|
||||
await tx.$executeRawUnsafe(`LOCK TABLE "${profileId}"."general" IN ACCESS EXCLUSIVE MODE`);
|
||||
ready(tx);
|
||||
await release;
|
||||
},
|
||||
{ timeout: 20_000 }
|
||||
);
|
||||
void locking.catch(lockError);
|
||||
try {
|
||||
const tx = await locked;
|
||||
const first = fetch(`${baseUrl}${server!.config.eventsPath}`, {
|
||||
headers: { authorization: `Bearer ${tokens[0]!}` },
|
||||
signal: aborts[0]!.signal,
|
||||
}).then(
|
||||
() => false,
|
||||
() => true
|
||||
);
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const rows = await tx.$queryRaw<Array<{ count: number }>>`
|
||||
SELECT COUNT(*)::int AS count FROM pg_locks
|
||||
WHERE NOT granted AND relation = ${`${profileId}.general`}::regclass
|
||||
AND database = (SELECT oid FROM pg_database WHERE datname = current_database())
|
||||
`;
|
||||
return rows[0]!.count;
|
||||
},
|
||||
{ timeout: 3000 }
|
||||
)
|
||||
.toBeGreaterThan(0);
|
||||
const startedAt = Date.now();
|
||||
const waiting = tokens.slice(1, 32).map((token, index) =>
|
||||
fetch(`${baseUrl}${server!.config.eventsPath}`, {
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
signal: aborts[index + 1]!.signal,
|
||||
})
|
||||
);
|
||||
aborts[0]!.abort();
|
||||
expect(await first).toBe(true);
|
||||
const responses = await Promise.all(waiting);
|
||||
expect(Date.now() - startedAt).toBeGreaterThanOrEqual(4500);
|
||||
expect(Date.now() - startedAt).toBeLessThan(8500);
|
||||
for (const response of responses) {
|
||||
expect(response.status).toBe(503);
|
||||
expect(await response.json()).toEqual({ ok: false, error: 'realtime_unavailable' });
|
||||
}
|
||||
const refusedAt = Date.now();
|
||||
const refused = await fetch(`${baseUrl}${server!.config.eventsPath}`, {
|
||||
headers: { authorization: `Bearer ${tokens[32]!}` },
|
||||
signal: aborts[32]!.signal,
|
||||
});
|
||||
expect(refused.status).toBe(503);
|
||||
expect(refused.headers.get('retry-after')).toBe('15');
|
||||
expect(Date.now() - refusedAt).toBeLessThan(2000);
|
||||
await refused.arrayBuffer();
|
||||
unlock();
|
||||
await locking;
|
||||
let recovered: Response | undefined;
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const response = await fetch(`${baseUrl}${server!.config.eventsPath}`, {
|
||||
headers: { authorization: `Bearer ${tokens[32]!}` },
|
||||
signal: aborts[32]!.signal,
|
||||
});
|
||||
if (response.status === 200) recovered = response;
|
||||
else await response.arrayBuffer();
|
||||
return response.status;
|
||||
})
|
||||
.toBe(200);
|
||||
const frame = await recovered!.body!.getReader().read();
|
||||
expect(new TextDecoder().decode(frame.value)).toContain('event: ready');
|
||||
} finally {
|
||||
unlock();
|
||||
for (const abort of aborts) abort.abort();
|
||||
await locking;
|
||||
await Promise.all(tokens.map((token) => accessTokenStore.revoke(token)));
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it('SSE accepts header auth and rejects query or cross-profile credentials', async () => {
|
||||
const token = await createAccessToken('sse-query', {});
|
||||
const queryOnly = await fetch(`${baseUrl}${server!.config.eventsPath}?token=${encodeURIComponent(token)}`);
|
||||
|
||||
Reference in New Issue
Block a user