Merge branch 'main' into feature/legacy-db-migration
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
|
||||
import { createGamePostgresConnector, GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
|
||||
|
||||
@@ -46,6 +47,8 @@ export class DatabaseTurnDaemonLease {
|
||||
private readonly heartbeatEnabled: boolean;
|
||||
private token: TurnDaemonLeaseToken | null = null;
|
||||
private heartbeatTimer: NodeJS.Timeout | null = null;
|
||||
private expiryTimer: NodeJS.Timeout | null = null;
|
||||
private renewalInFlight = false;
|
||||
private lost = false;
|
||||
|
||||
private constructor(
|
||||
@@ -71,6 +74,7 @@ export class DatabaseTurnDaemonLease {
|
||||
}
|
||||
|
||||
async acquire(): Promise<TurnDaemonLeaseToken | null> {
|
||||
const requestStartedAt = performance.now();
|
||||
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
|
||||
INSERT INTO "turn_daemon_lease" (
|
||||
"profile",
|
||||
@@ -111,6 +115,7 @@ export class DatabaseTurnDaemonLease {
|
||||
fencingEpoch: BigInt(row.fencing_epoch),
|
||||
};
|
||||
this.lost = false;
|
||||
this.scheduleExpiryWatchdog(requestStartedAt);
|
||||
if (this.heartbeatEnabled) {
|
||||
this.startHeartbeat();
|
||||
}
|
||||
@@ -127,26 +132,36 @@ export class DatabaseTurnDaemonLease {
|
||||
|
||||
async renew(): Promise<boolean> {
|
||||
const token = this.token;
|
||||
if (!token || this.lost) {
|
||||
if (!token || this.lost || this.renewalInFlight) {
|
||||
return false;
|
||||
}
|
||||
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
|
||||
UPDATE "turn_daemon_lease"
|
||||
SET
|
||||
"lease_until" = CURRENT_TIMESTAMP + (${this.leaseDurationMs} * INTERVAL '1 millisecond'),
|
||||
"heartbeat_at" = CURRENT_TIMESTAMP
|
||||
WHERE
|
||||
"profile" = ${token.profile}
|
||||
AND "owner_id" = ${token.ownerId}
|
||||
AND "fencing_epoch" = ${token.fencingEpoch}
|
||||
AND "lease_until" > CURRENT_TIMESTAMP
|
||||
RETURNING "profile", "owner_id", "fencing_epoch"
|
||||
`);
|
||||
if (rows.length === 0) {
|
||||
this.markLost();
|
||||
return false;
|
||||
const requestStartedAt = performance.now();
|
||||
this.renewalInFlight = true;
|
||||
try {
|
||||
const rows = await this.db.$queryRaw<LeaseRow[]>(GamePrisma.sql`
|
||||
UPDATE "turn_daemon_lease"
|
||||
SET
|
||||
"lease_until" = CURRENT_TIMESTAMP + (${this.leaseDurationMs} * INTERVAL '1 millisecond'),
|
||||
"heartbeat_at" = CURRENT_TIMESTAMP
|
||||
WHERE
|
||||
"profile" = ${token.profile}
|
||||
AND "owner_id" = ${token.ownerId}
|
||||
AND "fencing_epoch" = ${token.fencingEpoch}
|
||||
AND "lease_until" > CURRENT_TIMESTAMP
|
||||
RETURNING "profile", "owner_id", "fencing_epoch"
|
||||
`);
|
||||
if (rows.length === 0) {
|
||||
this.markLost();
|
||||
return false;
|
||||
}
|
||||
if (this.lost) {
|
||||
return false;
|
||||
}
|
||||
this.scheduleExpiryWatchdog(requestStartedAt);
|
||||
return true;
|
||||
} finally {
|
||||
this.renewalInFlight = false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async assertActive(transaction?: GamePrisma.TransactionClient): Promise<void> {
|
||||
@@ -173,6 +188,7 @@ export class DatabaseTurnDaemonLease {
|
||||
|
||||
async release(): Promise<void> {
|
||||
this.stopHeartbeat();
|
||||
this.stopExpiryWatchdog();
|
||||
const token = this.token;
|
||||
this.token = null;
|
||||
if (!token || this.lost) {
|
||||
@@ -216,8 +232,27 @@ export class DatabaseTurnDaemonLease {
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleExpiryWatchdog(requestStartedAt: number): void {
|
||||
// DB가 lease_until을 정하는 시점보다 앞선 요청 시작 시각을 기준으로
|
||||
// 잡아, heartbeat 응답이 멈춰도 DB lease 만료보다 늦게 pause하지 않는다.
|
||||
this.stopExpiryWatchdog();
|
||||
const remainingMs = Math.max(0, this.leaseDurationMs - (performance.now() - requestStartedAt));
|
||||
this.expiryTimer = setTimeout(() => {
|
||||
this.markLost();
|
||||
}, remainingMs);
|
||||
this.expiryTimer.unref();
|
||||
}
|
||||
|
||||
private stopExpiryWatchdog(): void {
|
||||
if (this.expiryTimer) {
|
||||
clearTimeout(this.expiryTimer);
|
||||
this.expiryTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private markLost(): void {
|
||||
this.lost = true;
|
||||
this.stopHeartbeat();
|
||||
this.stopExpiryWatchdog();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { DatabaseTurnDaemonLease } from '../../src/lifecycle/databaseTurnDaemonLease.ts';
|
||||
|
||||
const databaseUrl = process.env.TURN_DAEMON_LEASE_DATABASE_URL;
|
||||
const profile = process.env.TURN_DAEMON_LEASE_PROFILE;
|
||||
const ownerId = process.env.TURN_DAEMON_LEASE_OWNER_ID;
|
||||
const leaseDurationMs = Number(process.env.TURN_DAEMON_LEASE_DURATION_MS);
|
||||
|
||||
if (!databaseUrl || !profile || !ownerId || !Number.isInteger(leaseDurationMs)) {
|
||||
throw new Error('lease holder requires database URL, profile, owner, and duration');
|
||||
}
|
||||
|
||||
const lease = await DatabaseTurnDaemonLease.connect(databaseUrl, {
|
||||
profile,
|
||||
ownerId,
|
||||
leaseDurationMs,
|
||||
heartbeat: true,
|
||||
});
|
||||
const token = await lease.acquire();
|
||||
if (!token) {
|
||||
throw new Error('lease holder could not acquire the profile lease');
|
||||
}
|
||||
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
profile: token.profile,
|
||||
ownerId: token.ownerId,
|
||||
fencingEpoch: token.fencingEpoch.toString(),
|
||||
})}\n`
|
||||
);
|
||||
|
||||
setInterval(() => undefined, 60_000);
|
||||
@@ -1,3 +1,8 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { once } from 'node:events';
|
||||
import { createConnection, createServer, type Server, type Socket } from 'node:net';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
import { createGamePostgresConnector, type GamePrisma, type GamePrismaClient } from '@sammo-ts/infra';
|
||||
@@ -7,6 +12,153 @@ import { DatabaseTurnDaemonLease, TurnDaemonLeaseLostError } from '../src/lifecy
|
||||
const databaseUrl = process.env.TURN_DAEMON_LEASE_DATABASE_URL ?? process.env.INPUT_EVENT_DATABASE_URL;
|
||||
const integration = describe.skipIf(!databaseUrl);
|
||||
const profilePrefix = 'integration:turn-lease:';
|
||||
const holderScript = fileURLToPath(new URL('./helpers/turnDaemonLeaseHolder.mjs', import.meta.url));
|
||||
|
||||
type HolderReady = {
|
||||
profile: string;
|
||||
ownerId: string;
|
||||
fencingEpoch: string;
|
||||
};
|
||||
|
||||
const waitForHolderReady = async (child: ReturnType<typeof spawn>, timeoutMs = 10_000): Promise<HolderReady> => {
|
||||
let output = '';
|
||||
const ready = new Promise<HolderReady>((resolve, reject) => {
|
||||
child.stdout?.setEncoding('utf8');
|
||||
child.stdout?.on('data', (chunk: string) => {
|
||||
output += chunk;
|
||||
const newline = output.indexOf('\n');
|
||||
if (newline < 0) {
|
||||
return;
|
||||
}
|
||||
resolve(JSON.parse(output.slice(0, newline)) as HolderReady);
|
||||
});
|
||||
child.once('error', reject);
|
||||
child.once('exit', (code, signal) => {
|
||||
reject(new Error(`lease holder exited before readiness: code=${String(code)} signal=${String(signal)}`));
|
||||
});
|
||||
});
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
setTimeout(() => reject(new Error('lease holder readiness timed out')), timeoutMs).unref();
|
||||
});
|
||||
return Promise.race([ready, timeout]);
|
||||
};
|
||||
|
||||
const delay = (durationMs: number): Promise<void> =>
|
||||
new Promise((resolve) => {
|
||||
setTimeout(resolve, durationMs);
|
||||
});
|
||||
|
||||
const createPartitionableTcpProxy = async (
|
||||
upstreamHost: string,
|
||||
upstreamPort: number
|
||||
): Promise<{
|
||||
port: number;
|
||||
partition: () => Promise<void>;
|
||||
restore: () => Promise<void>;
|
||||
close: () => Promise<void>;
|
||||
}> => {
|
||||
type ProxyPair = {
|
||||
client: Socket;
|
||||
upstream: Socket;
|
||||
};
|
||||
const pairs = new Set<ProxyPair>();
|
||||
let server: Server | null = null;
|
||||
let port = 0;
|
||||
let partitioned = false;
|
||||
|
||||
const resumePair = (pair: ProxyPair): void => {
|
||||
if (partitioned || pair.client.destroyed || pair.upstream.destroyed || pair.upstream.connecting) {
|
||||
return;
|
||||
}
|
||||
pair.client.pipe(pair.upstream);
|
||||
pair.upstream.pipe(pair.client);
|
||||
};
|
||||
|
||||
const listen = async (requestedPort: number): Promise<void> => {
|
||||
const nextServer = createServer((client) => {
|
||||
const upstream = createConnection({ host: upstreamHost, port: upstreamPort });
|
||||
const pair = { client, upstream };
|
||||
pairs.add(pair);
|
||||
client.on('error', () => undefined);
|
||||
upstream.on('error', () => undefined);
|
||||
client.once('close', () => {
|
||||
pairs.delete(pair);
|
||||
if (!upstream.destroyed) {
|
||||
upstream.destroy();
|
||||
}
|
||||
});
|
||||
upstream.once('close', () => {
|
||||
pairs.delete(pair);
|
||||
if (!client.destroyed) {
|
||||
client.destroy();
|
||||
}
|
||||
});
|
||||
upstream.once('connect', () => resumePair(pair));
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onError = (error: Error): void => reject(error);
|
||||
nextServer.once('error', onError);
|
||||
nextServer.listen(requestedPort, '127.0.0.1', () => {
|
||||
nextServer.off('error', onError);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
server = nextServer;
|
||||
const address = nextServer.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('TCP proxy did not expose a numeric port');
|
||||
}
|
||||
port = address.port;
|
||||
};
|
||||
|
||||
const partition = (): Promise<void> => {
|
||||
partitioned = true;
|
||||
for (const pair of pairs) {
|
||||
pair.client.unpipe(pair.upstream);
|
||||
pair.upstream.unpipe(pair.client);
|
||||
}
|
||||
return Promise.resolve();
|
||||
};
|
||||
|
||||
const close = async (): Promise<void> => {
|
||||
const activeServer = server;
|
||||
server = null;
|
||||
if (!activeServer) {
|
||||
return;
|
||||
}
|
||||
const closed = new Promise<void>((resolve, reject) => {
|
||||
activeServer.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
for (const pair of pairs) {
|
||||
pair.client.destroy();
|
||||
pair.upstream.destroy();
|
||||
}
|
||||
pairs.clear();
|
||||
await closed;
|
||||
};
|
||||
|
||||
await listen(0);
|
||||
return {
|
||||
get port() {
|
||||
return port;
|
||||
},
|
||||
partition,
|
||||
restore: () => {
|
||||
partitioned = false;
|
||||
for (const pair of pairs) {
|
||||
resumePair(pair);
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
close,
|
||||
};
|
||||
};
|
||||
|
||||
integration('database turn daemon lease and fencing', () => {
|
||||
let db: GamePrismaClient;
|
||||
@@ -115,4 +267,115 @@ integration('database turn daemon lease and fencing', () => {
|
||||
expect((await second.acquire())?.fencingEpoch).toBe(2n);
|
||||
await expect(first.assertActive()).rejects.toBeInstanceOf(TurnDaemonLeaseLostError);
|
||||
});
|
||||
|
||||
it('takes over after a heartbeat owner process is killed and its lease expires', async () => {
|
||||
const profile = `${profilePrefix}process-kill`;
|
||||
const leaseDurationMs = 1_200;
|
||||
const holder = spawn(process.execPath, ['--experimental-strip-types', holderScript], {
|
||||
env: {
|
||||
...process.env,
|
||||
TURN_DAEMON_LEASE_DATABASE_URL: databaseUrl!,
|
||||
TURN_DAEMON_LEASE_PROFILE: profile,
|
||||
TURN_DAEMON_LEASE_OWNER_ID: 'process-owner',
|
||||
TURN_DAEMON_LEASE_DURATION_MS: String(leaseDurationMs),
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let successor: DatabaseTurnDaemonLease | null = null;
|
||||
try {
|
||||
await expect(waitForHolderReady(holder)).resolves.toEqual({
|
||||
profile,
|
||||
ownerId: 'process-owner',
|
||||
fencingEpoch: '1',
|
||||
});
|
||||
expect(await db.turnDaemonLease.findUniqueOrThrow({ where: { profile } })).toMatchObject({
|
||||
ownerId: 'process-owner',
|
||||
fencingEpoch: 1n,
|
||||
});
|
||||
|
||||
expect(holder.kill('SIGKILL')).toBe(true);
|
||||
const [exitCode, signal] = (await once(holder, 'exit')) as [number | null, NodeJS.Signals | null];
|
||||
expect(exitCode).toBeNull();
|
||||
expect(signal).toBe('SIGKILL');
|
||||
|
||||
successor = await createLease(profile, 'successor-owner');
|
||||
expect(await successor.acquire()).toBeNull();
|
||||
|
||||
const deadline = Date.now() + leaseDurationMs * 4;
|
||||
let successorToken = null;
|
||||
while (Date.now() < deadline) {
|
||||
successorToken = await successor.acquire();
|
||||
if (successorToken) {
|
||||
break;
|
||||
}
|
||||
await delay(100);
|
||||
}
|
||||
expect(successorToken).toMatchObject({
|
||||
profile,
|
||||
ownerId: 'successor-owner',
|
||||
fencingEpoch: 2n,
|
||||
});
|
||||
expect(await successor.assertActive()).toBeUndefined();
|
||||
} finally {
|
||||
if (holder.exitCode === null && holder.signalCode === null) {
|
||||
holder.kill('SIGKILL');
|
||||
await once(holder, 'exit');
|
||||
}
|
||||
await successor?.release();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it('stays fenced after a database partition outlasts the heartbeat lease and connectivity recovers', async () => {
|
||||
const profile = `${profilePrefix}database-partition`;
|
||||
const leaseDurationMs = 1_800;
|
||||
const directUrl = new URL(databaseUrl!);
|
||||
const upstreamPort = Number(directUrl.port || '5432');
|
||||
const proxy = await createPartitionableTcpProxy(directUrl.hostname, upstreamPort);
|
||||
directUrl.hostname = '127.0.0.1';
|
||||
directUrl.port = String(proxy.port);
|
||||
const partitionedOwner = await DatabaseTurnDaemonLease.connect(directUrl.toString(), {
|
||||
profile,
|
||||
ownerId: 'partitioned-owner',
|
||||
leaseDurationMs,
|
||||
heartbeat: true,
|
||||
});
|
||||
leases.push(partitionedOwner);
|
||||
let successor: DatabaseTurnDaemonLease | null = null;
|
||||
try {
|
||||
expect((await partitionedOwner.acquire())?.fencingEpoch).toBe(1n);
|
||||
|
||||
await proxy.partition();
|
||||
successor = await createLease(profile, 'partition-successor');
|
||||
expect(await successor.acquire()).toBeNull();
|
||||
|
||||
const lostDeadline = Date.now() + leaseDurationMs * 2;
|
||||
while (!partitionedOwner.isLost() && Date.now() < lostDeadline) {
|
||||
await delay(50);
|
||||
}
|
||||
expect(partitionedOwner.isLost()).toBe(true);
|
||||
|
||||
const takeoverDeadline = Date.now() + leaseDurationMs * 4;
|
||||
let successorToken = null;
|
||||
while (Date.now() < takeoverDeadline) {
|
||||
successorToken = await successor.acquire();
|
||||
if (successorToken) {
|
||||
break;
|
||||
}
|
||||
await delay(100);
|
||||
}
|
||||
expect(successorToken).toMatchObject({
|
||||
profile,
|
||||
ownerId: 'partition-successor',
|
||||
fencingEpoch: 2n,
|
||||
});
|
||||
|
||||
await proxy.restore();
|
||||
expect(await partitionedOwner.renew()).toBe(false);
|
||||
await expect(partitionedOwner.assertActive()).rejects.toBeInstanceOf(TurnDaemonLeaseLostError);
|
||||
await expect(successor.assertActive()).resolves.toBeUndefined();
|
||||
} finally {
|
||||
await proxy.close();
|
||||
await successor?.release();
|
||||
}
|
||||
}, 20_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
import fastify, { type FastifyRequest } from 'fastify';
|
||||
import { fastifyTRPCPlugin } from '@trpc/server/adapters/fastify';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import type { GatewayPrismaClient } from '@sammo-ts/infra';
|
||||
|
||||
import { InMemoryGatewaySessionService } from '../src/auth/inMemorySessionService.js';
|
||||
import { createInMemoryUserRepository } from '../src/auth/inMemoryUserRepository.js';
|
||||
import { createPasswordEnvelopeService } from '../src/auth/passwordEnvelope.js';
|
||||
import { createGatewayApiContext } from '../src/context.js';
|
||||
import { InMemoryProfileStatusService } from '../src/lobby/profileStatusService.js';
|
||||
import type { GatewayProfileRepository } from '../src/orchestrator/profileRepository.js';
|
||||
import { appRouter } from '../src/router.js';
|
||||
|
||||
const profile = {
|
||||
profileName: 'che:default',
|
||||
profile: 'che',
|
||||
scenario: 'default',
|
||||
apiPort: 15003,
|
||||
status: 'RUNNING' as const,
|
||||
buildStatus: 'SUCCEEDED' as const,
|
||||
meta: {},
|
||||
createdAt: '2026-07-26T00:00:00.000Z',
|
||||
updatedAt: '2026-07-26T00:00:00.000Z',
|
||||
};
|
||||
|
||||
const profiles: GatewayProfileRepository = {
|
||||
listProfiles: async () => [profile],
|
||||
getProfile: async (profileName) => (profileName === profile.profileName ? profile : null),
|
||||
upsertProfile: async () => profile,
|
||||
updateScenario: async () => profile,
|
||||
updateStatus: async () => profile,
|
||||
updateBuildStatus: async () => profile,
|
||||
updateMeta: async () => profile,
|
||||
listReservedToStart: async () => [],
|
||||
findQueuedBuild: async () => null,
|
||||
updateLastError: async () => {},
|
||||
updateWorkspaceUsage: async () => {},
|
||||
clearWorkspaceUsage: async () => {},
|
||||
listOperations: async () => [],
|
||||
getOperation: async () => null,
|
||||
createOperation: async () => {
|
||||
throw new Error('not used');
|
||||
},
|
||||
claimNextOperation: async () => null,
|
||||
completeOperation: async () => {
|
||||
throw new Error('not used');
|
||||
},
|
||||
requeueOperation: async () => {
|
||||
throw new Error('not used');
|
||||
},
|
||||
cancelOperation: async () => false,
|
||||
retryOperation: async () => null,
|
||||
};
|
||||
|
||||
const openApps = new Set<ReturnType<typeof fastify>>();
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.allSettled(Array.from(openApps, (app) => app.close()));
|
||||
openApps.clear();
|
||||
});
|
||||
|
||||
const createHarness = async (adminRoles = ['user', 'admin.users.manage', 'admin.survey.open:che:default']) => {
|
||||
const users = createInMemoryUserRepository();
|
||||
const admin = await users.createUser({
|
||||
username: 'scoped-admin',
|
||||
password: 'secretpass',
|
||||
displayName: 'Scoped Admin',
|
||||
});
|
||||
await users.updateRoles(admin.id, adminRoles);
|
||||
const target = await users.createUser({
|
||||
username: 'target-user',
|
||||
password: 'secretpass',
|
||||
displayName: 'Target',
|
||||
});
|
||||
const sessions = new InMemoryGatewaySessionService({
|
||||
sessionTtlSeconds: 600,
|
||||
gameSessionTtlSeconds: 600,
|
||||
});
|
||||
const adminSession = await sessions.createSession({ ...admin, roles: adminRoles });
|
||||
const targetSession = await sessions.createSession(target);
|
||||
const flushes: Array<{ userId: string; reason?: string }> = [];
|
||||
const app = fastify({ logger: false });
|
||||
await app.register(fastifyTRPCPlugin, {
|
||||
prefix: '/trpc',
|
||||
trpcOptions: {
|
||||
router: appRouter,
|
||||
createContext: ({ req }: { req: FastifyRequest }) =>
|
||||
createGatewayApiContext({
|
||||
users,
|
||||
sessions,
|
||||
flushPublisher: {
|
||||
publishUserFlush: async (userId, reason) => {
|
||||
flushes.push({ userId, reason });
|
||||
},
|
||||
},
|
||||
gameTokenSecret: 'transport-e2e-secret',
|
||||
gameSessionTtlSeconds: 600,
|
||||
kakaoClient: {} as never,
|
||||
oauthSessions: {} as never,
|
||||
publicBaseUrl: 'http://127.0.0.1',
|
||||
adminLocalAccountEnabled: false,
|
||||
localRegistrationEnabled: true,
|
||||
localAccountGraceDays: 7,
|
||||
passwordEnvelope: createPasswordEnvelopeService(),
|
||||
profiles,
|
||||
orchestrator: {
|
||||
start: () => {},
|
||||
stop: async () => {},
|
||||
reconcileNow: async () => {},
|
||||
runScheduleNow: async () => {},
|
||||
runBuildQueueNow: async () => {},
|
||||
runOperationsNow: async () => {},
|
||||
cleanupStaleWorkspaces: async () => ({ removed: [], skipped: [] }),
|
||||
listRuntimeStates: async () => [],
|
||||
},
|
||||
profileStatus: new InMemoryProfileStatusService(),
|
||||
requestHeaders: req.headers,
|
||||
prisma: {
|
||||
appUser: {
|
||||
findFirst: async () => ({ id: 'bootstrap-user' }),
|
||||
},
|
||||
} as unknown as GatewayPrismaClient,
|
||||
}),
|
||||
},
|
||||
});
|
||||
const baseUrl = await app.listen({ host: '127.0.0.1', port: 0 });
|
||||
openApps.add(app);
|
||||
return {
|
||||
app,
|
||||
baseUrl,
|
||||
users,
|
||||
admin,
|
||||
target,
|
||||
adminSessionToken: adminSession.sessionToken,
|
||||
targetSessionToken: targetSession.sessionToken,
|
||||
flushes,
|
||||
};
|
||||
};
|
||||
|
||||
const postTrpc = async (
|
||||
baseUrl: string,
|
||||
procedure: string,
|
||||
input: unknown,
|
||||
sessionToken?: string
|
||||
): Promise<{ response: Response; body: unknown }> => {
|
||||
const response = await fetch(`${baseUrl}/trpc/${procedure}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(sessionToken ? { 'x-session-token': sessionToken } : {}),
|
||||
},
|
||||
body: JSON.stringify(input),
|
||||
});
|
||||
return {
|
||||
response,
|
||||
body: (await response.json()) as unknown,
|
||||
};
|
||||
};
|
||||
|
||||
describe('admin security over HTTP transport', () => {
|
||||
it('accepts an equal scoped role and rejects wildcard escalation without mutating roles', async () => {
|
||||
const harness = await createHarness();
|
||||
|
||||
const allowed = await postTrpc(
|
||||
harness.baseUrl,
|
||||
'admin.users.updateRoles',
|
||||
{
|
||||
userId: harness.target.id,
|
||||
roles: ['admin.survey.open:che:default'],
|
||||
mode: 'grant',
|
||||
},
|
||||
harness.adminSessionToken
|
||||
);
|
||||
if (allowed.response.status !== 200) {
|
||||
throw new Error(`allowed role request failed: ${JSON.stringify(allowed.body)}`);
|
||||
}
|
||||
expect(allowed.body).toMatchObject({
|
||||
result: {
|
||||
data: {
|
||||
roles: ['user', 'admin.survey.open:che:default'],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const rejected = await postTrpc(
|
||||
harness.baseUrl,
|
||||
'admin.users.updateRoles',
|
||||
{
|
||||
userId: harness.target.id,
|
||||
roles: ['admin.survey.open:*'],
|
||||
mode: 'grant',
|
||||
},
|
||||
harness.adminSessionToken
|
||||
);
|
||||
expect(rejected.response.status).toBe(403);
|
||||
expect(rejected.body).toMatchObject({
|
||||
error: {
|
||||
data: {
|
||||
code: 'FORBIDDEN',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect((await harness.users.findById(harness.target.id))?.roles).toEqual([
|
||||
'user',
|
||||
'admin.survey.open:che:default',
|
||||
]);
|
||||
});
|
||||
|
||||
it('rejects an unauthenticated role change at the HTTP header boundary', async () => {
|
||||
const harness = await createHarness();
|
||||
|
||||
const rejected = await postTrpc(harness.baseUrl, 'admin.users.updateRoles', {
|
||||
userId: harness.target.id,
|
||||
roles: ['admin.survey.open:che:default'],
|
||||
mode: 'grant',
|
||||
});
|
||||
|
||||
expect(rejected.response.status).toBe(401);
|
||||
expect(rejected.body).toMatchObject({
|
||||
error: {
|
||||
data: {
|
||||
code: 'UNAUTHORIZED',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect((await harness.users.findById(harness.target.id))?.roles).toEqual(['user']);
|
||||
});
|
||||
|
||||
it('rejects self-escalation and set-mode removal outside a scoped administrator role', async () => {
|
||||
const harness = await createHarness();
|
||||
|
||||
const selfEscalation = await postTrpc(
|
||||
harness.baseUrl,
|
||||
'admin.users.updateRoles',
|
||||
{
|
||||
userId: harness.admin.id,
|
||||
roles: ['admin.survey.open:*'],
|
||||
mode: 'grant',
|
||||
},
|
||||
harness.adminSessionToken
|
||||
);
|
||||
expect(selfEscalation.response.status).toBe(403);
|
||||
expect((await harness.users.findByUsername('scoped-admin'))?.roles).toEqual([
|
||||
'user',
|
||||
'admin.users.manage',
|
||||
'admin.survey.open:che:default',
|
||||
]);
|
||||
|
||||
await harness.users.updateRoles(harness.target.id, ['user', 'admin.survey.open:*']);
|
||||
const outOfScopeRemoval = await postTrpc(
|
||||
harness.baseUrl,
|
||||
'admin.users.updateRoles',
|
||||
{
|
||||
userId: harness.target.id,
|
||||
roles: ['user'],
|
||||
mode: 'set',
|
||||
},
|
||||
harness.adminSessionToken
|
||||
);
|
||||
expect(outOfScopeRemoval.response.status).toBe(403);
|
||||
expect((await harness.users.findById(harness.target.id))?.roles).toEqual(['user', 'admin.survey.open:*']);
|
||||
});
|
||||
|
||||
it('allows a superuser to grant a root role over HTTP', async () => {
|
||||
const harness = await createHarness(['user', 'superuser']);
|
||||
|
||||
const granted = await postTrpc(
|
||||
harness.baseUrl,
|
||||
'admin.users.updateRoles',
|
||||
{
|
||||
userId: harness.target.id,
|
||||
roles: ['superuser'],
|
||||
mode: 'grant',
|
||||
},
|
||||
harness.adminSessionToken
|
||||
);
|
||||
|
||||
expect(granted.response.status).toBe(200);
|
||||
expect((await harness.users.findById(harness.target.id))?.roles).toEqual(['user', 'superuser']);
|
||||
});
|
||||
|
||||
it('blocks game-session issuance after an HTTP sanction update while retaining the gateway session', async () => {
|
||||
const harness = await createHarness();
|
||||
|
||||
const updated = await postTrpc(
|
||||
harness.baseUrl,
|
||||
'admin.users.updateSanctions',
|
||||
{
|
||||
userId: harness.target.id,
|
||||
patch: {
|
||||
suspendedUntil: '2099-01-01T00:00:00.000Z',
|
||||
},
|
||||
},
|
||||
harness.adminSessionToken
|
||||
);
|
||||
if (updated.response.status !== 200) {
|
||||
throw new Error(`sanction request failed: ${JSON.stringify(updated.body)}`);
|
||||
}
|
||||
expect(harness.flushes).toEqual([
|
||||
{
|
||||
userId: harness.target.id,
|
||||
reason: 'admin-sanctions-updated',
|
||||
},
|
||||
]);
|
||||
|
||||
const blocked = await postTrpc(harness.baseUrl, 'auth.issueGameSession', {
|
||||
sessionToken: harness.targetSessionToken,
|
||||
profile: profile.profileName,
|
||||
});
|
||||
expect(blocked.response.status).toBe(403);
|
||||
expect(blocked.body).toMatchObject({
|
||||
error: {
|
||||
data: {
|
||||
code: 'FORBIDDEN',
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -119,6 +119,7 @@ integration('live sortie PostgreSQL persistence retry', () => {
|
||||
|
||||
it.each([
|
||||
['conquest and nation collapse', 'live-sortie-conquest.json'],
|
||||
['collapsed nation conflict cleanup', 'live-sortie-collapse-conflict.json'],
|
||||
['single defending general', 'live-sortie-defender.json'],
|
||||
['multiple defending generals', 'live-sortie-multiple-defenders.json'],
|
||||
['supply retreat', 'live-sortie-supply-retreat.json'],
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { loadActionModuleBundle, type City, type GeneralActionModule, type MapDefinition } from '@sammo-ts/logic';
|
||||
import { InMemoryTurnWorld } from '@sammo-ts/game-engine/turn/inMemoryWorld.js';
|
||||
import { createRaiseDisasterHandler } from '@sammo-ts/game-engine/turn/monthlyDisasterAction.js';
|
||||
import type { TurnEvent, TurnGeneral, TurnWorldSnapshot, TurnWorldState } from '@sammo-ts/game-engine/turn/types.js';
|
||||
|
||||
import { findTurnDifferentialWorkspaceRoot } from '../src/turn-differential/referenceSnapshot.js';
|
||||
|
||||
type ReferenceGeneral = {
|
||||
id: number;
|
||||
injury: number;
|
||||
crew: number;
|
||||
train: number;
|
||||
atmos: number;
|
||||
item: string;
|
||||
};
|
||||
|
||||
type ReferenceCity = {
|
||||
id: number;
|
||||
state: number;
|
||||
population: number;
|
||||
agriculture: number;
|
||||
commerce: number;
|
||||
security: number;
|
||||
trust: number;
|
||||
defence: number;
|
||||
wall: number;
|
||||
};
|
||||
|
||||
type ReferenceLog = {
|
||||
generalId?: number;
|
||||
text: string;
|
||||
scope?: string;
|
||||
category?: string;
|
||||
};
|
||||
|
||||
type ReferenceMonthlyTrace = {
|
||||
action: string;
|
||||
before: {
|
||||
watermarks: {
|
||||
logId: number;
|
||||
};
|
||||
};
|
||||
after: {
|
||||
logs: ReferenceLog[];
|
||||
};
|
||||
beforeDetails: {
|
||||
generals: ReferenceGeneral[];
|
||||
cities: ReferenceCity[];
|
||||
worldHistory: ReferenceLog[];
|
||||
};
|
||||
afterDetails: {
|
||||
generals: ReferenceGeneral[];
|
||||
cities: ReferenceCity[];
|
||||
worldHistory: ReferenceLog[];
|
||||
};
|
||||
};
|
||||
|
||||
const configuredWorkspaceRoot = process.env.TURN_DIFFERENTIAL_WORKSPACE_ROOT;
|
||||
const workspaceRoot = configuredWorkspaceRoot ?? findTurnDifferentialWorkspaceRoot(process.cwd());
|
||||
const integration = describe.skipIf(!workspaceRoot || process.env.TURN_DIFFERENTIAL_REFERENCE !== '1');
|
||||
|
||||
const fixtureRelativePath = 'fixtures/monthly-differential/raise-disaster-january.json';
|
||||
|
||||
const runReferenceMonthlyTrace = (): ReferenceMonthlyTrace => {
|
||||
const stackDirectory = path.join(workspaceRoot!, 'docker_compose_files/reference');
|
||||
const fixture = fs.readFileSync(path.join(stackDirectory, fixtureRelativePath), 'utf8');
|
||||
const runnerScript =
|
||||
process.env.MONTHLY_DIFFERENTIAL_RUNNER_SCRIPT ??
|
||||
path.join(workspaceRoot!, 'ref/sam/hwe/compare/monthly_event_trace.php');
|
||||
const stdout = execFileSync('./scripts/run-turn-differential-case.sh', ['-'], {
|
||||
cwd: stackDirectory,
|
||||
input: fixture,
|
||||
encoding: 'utf8',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
env: {
|
||||
...process.env,
|
||||
TURN_DIFFERENTIAL_RUNNER_SCRIPT: runnerScript,
|
||||
},
|
||||
});
|
||||
return JSON.parse(stdout) as ReferenceMonthlyTrace;
|
||||
};
|
||||
|
||||
const map: MapDefinition = {
|
||||
id: 'monthly-disaster-reference',
|
||||
name: 'monthly-disaster-reference',
|
||||
cities: [],
|
||||
defaults: { trust: 50, trade: 100, supplyState: 1, frontState: 0 },
|
||||
};
|
||||
|
||||
const city: City = {
|
||||
id: 1,
|
||||
name: '재난도시',
|
||||
nationId: 0,
|
||||
level: 1,
|
||||
state: 5,
|
||||
population: 1_001,
|
||||
populationMax: 2_000,
|
||||
agriculture: 501,
|
||||
agricultureMax: 1_000,
|
||||
commerce: 499,
|
||||
commerceMax: 1_000,
|
||||
security: 0,
|
||||
securityMax: 100,
|
||||
supplyState: 1,
|
||||
frontState: 2,
|
||||
defence: 99,
|
||||
defenceMax: 1_000,
|
||||
wall: 101,
|
||||
wallMax: 1_000,
|
||||
conflict: {},
|
||||
meta: { trust: 99.5, trade: 100 },
|
||||
};
|
||||
|
||||
const buildGeneral = (id: number, name: string, injury: number, item: string | null): TurnGeneral => ({
|
||||
id,
|
||||
name,
|
||||
nationId: 0,
|
||||
cityId: 1,
|
||||
troopId: 0,
|
||||
stats: { leadership: 50, strength: 50, intelligence: 50 },
|
||||
experience: 0,
|
||||
dedication: 0,
|
||||
officerLevel: 1,
|
||||
role: {
|
||||
personality: null,
|
||||
specialDomestic: null,
|
||||
specialWar: null,
|
||||
items: { horse: null, weapon: null, book: null, item },
|
||||
},
|
||||
injury,
|
||||
gold: 1_000,
|
||||
rice: 1_000,
|
||||
crew: 99,
|
||||
crewTypeId: 1_100,
|
||||
train: 51,
|
||||
atmos: 50,
|
||||
age: 20,
|
||||
npcState: 0,
|
||||
triggerState: { flags: {}, counters: {}, modifiers: {}, meta: {} },
|
||||
meta: { killturn: 24 },
|
||||
turnTime: new Date('0193-01-01T00:00:00.000Z'),
|
||||
});
|
||||
|
||||
const projectReferenceGenerals = (generals: ReferenceGeneral[]) =>
|
||||
generals.map((general) => ({
|
||||
id: general.id,
|
||||
injury: general.injury,
|
||||
crew: general.crew,
|
||||
train: general.train,
|
||||
atmos: general.atmos,
|
||||
item: general.item === 'None' ? null : general.item,
|
||||
}));
|
||||
|
||||
const projectCoreGenerals = (world: InMemoryTurnWorld) =>
|
||||
world
|
||||
.listGenerals()
|
||||
.sort((left, right) => left.id - right.id)
|
||||
.map((general) => ({
|
||||
id: general.id,
|
||||
injury: general.injury,
|
||||
crew: general.crew,
|
||||
train: general.train,
|
||||
atmos: general.atmos,
|
||||
item: general.role.items.item,
|
||||
}));
|
||||
|
||||
const projectReferenceCity = (citySnapshot: ReferenceCity) => ({
|
||||
state: citySnapshot.state,
|
||||
population: citySnapshot.population,
|
||||
agriculture: citySnapshot.agriculture,
|
||||
commerce: citySnapshot.commerce,
|
||||
security: citySnapshot.security,
|
||||
trust: citySnapshot.trust,
|
||||
defence: citySnapshot.defence,
|
||||
wall: citySnapshot.wall,
|
||||
});
|
||||
|
||||
const projectCoreCity = (citySnapshot: City) => ({
|
||||
state: citySnapshot.state,
|
||||
population: citySnapshot.population,
|
||||
agriculture: citySnapshot.agriculture,
|
||||
commerce: citySnapshot.commerce,
|
||||
security: citySnapshot.security,
|
||||
trust: citySnapshot.meta.trust,
|
||||
defence: citySnapshot.defence,
|
||||
wall: citySnapshot.wall,
|
||||
});
|
||||
|
||||
const normalizeStoredLogText = (value: string): string =>
|
||||
value.replace(/^(?:<C>●<\/>|<S>◆<\/>|<R>★<\/>)(?:(?:\d+년 )?\d+월:|\d+년:)?/, '');
|
||||
|
||||
const projectReferenceLogs = (trace: ReferenceMonthlyTrace) => [
|
||||
...trace.afterDetails.worldHistory.map((entry) => ({
|
||||
scope: 'system',
|
||||
category: 'history',
|
||||
generalId: null,
|
||||
text: normalizeStoredLogText(entry.text),
|
||||
})),
|
||||
...trace.after.logs
|
||||
.filter((entry) => entry.generalId !== undefined)
|
||||
.map((entry) => ({
|
||||
scope: entry.scope,
|
||||
category: entry.category,
|
||||
generalId: entry.generalId ?? null,
|
||||
text: normalizeStoredLogText(entry.text),
|
||||
})),
|
||||
];
|
||||
|
||||
const buildCoreWorld = async (generalActionModules: GeneralActionModule[]): Promise<InMemoryTurnWorld> => {
|
||||
const state: TurnWorldState = {
|
||||
id: 1,
|
||||
currentYear: 193,
|
||||
currentMonth: 1,
|
||||
tickSeconds: 600,
|
||||
lastTurnTime: new Date('0193-01-01T00:00:00.000Z'),
|
||||
meta: { hiddenSeed: 'disaster-test-6' },
|
||||
};
|
||||
const snapshot: TurnWorldSnapshot = {
|
||||
scenarioConfig: {
|
||||
stat: { total: 300, min: 10, max: 100, npcTotal: 150, npcMax: 50, npcMin: 10, chiefMin: 70 },
|
||||
iconPath: '',
|
||||
map: {},
|
||||
const: {},
|
||||
environment: { mapName: map.id, unitSet: 'default' },
|
||||
},
|
||||
scenarioMeta: {
|
||||
title: 'monthly disaster reference',
|
||||
startYear: 190,
|
||||
life: null,
|
||||
fiction: null,
|
||||
history: [],
|
||||
ignoreDefaultEvents: false,
|
||||
},
|
||||
map,
|
||||
diplomacy: [],
|
||||
events: [],
|
||||
initialEvents: [],
|
||||
generals: [
|
||||
buildGeneral(1, '보호장수', 0, 'che_부적_태현청생부'),
|
||||
buildGeneral(3, '부상장수', 0, null),
|
||||
buildGeneral(5, '중상장수', 70, null),
|
||||
],
|
||||
cities: [city],
|
||||
nations: [],
|
||||
troops: [],
|
||||
};
|
||||
const world = new InMemoryTurnWorld(state, snapshot, {
|
||||
schedule: { entries: [{ startMinute: 0, tickMinutes: 10 }] },
|
||||
});
|
||||
const event: TurnEvent = {
|
||||
id: 1,
|
||||
targetCode: 'month',
|
||||
priority: 1_000,
|
||||
condition: true,
|
||||
action: [['RaiseDisaster']],
|
||||
meta: {},
|
||||
};
|
||||
await createRaiseDisasterHandler({
|
||||
getWorld: () => world,
|
||||
generalActionModules,
|
||||
})([], { year: 193, month: 1, startyear: 190, currentEventID: event.id, turnTime: state.lastTurnTime }, event);
|
||||
return world;
|
||||
};
|
||||
|
||||
integration('RaiseDisaster core ↔ legacy database snapshot', () => {
|
||||
it('matches city damage, item-aware injury, and persisted logs', async () => {
|
||||
const reference = runReferenceMonthlyTrace();
|
||||
expect(reference.action).toBe('RaiseDisaster');
|
||||
expect(projectReferenceGenerals(reference.beforeDetails.generals)).toEqual([
|
||||
{ id: 1, injury: 0, crew: 99, train: 51, atmos: 50, item: 'che_부적_태현청생부' },
|
||||
{ id: 3, injury: 0, crew: 99, train: 51, atmos: 50, item: null },
|
||||
{ id: 5, injury: 70, crew: 99, train: 51, atmos: 50, item: null },
|
||||
]);
|
||||
|
||||
const moduleBundle = await loadActionModuleBundle();
|
||||
const core = await buildCoreWorld(moduleBundle.general);
|
||||
const coreCity = projectCoreCity(core.getCityById(1)!);
|
||||
const referenceCity = projectReferenceCity(reference.afterDetails.cities[0]!);
|
||||
expect({ ...coreCity, trust: undefined }).toEqual({ ...referenceCity, trust: undefined });
|
||||
expect(coreCity.trust).toBeCloseTo(referenceCity.trust);
|
||||
expect(projectCoreGenerals(core)).toEqual(projectReferenceGenerals(reference.afterDetails.generals));
|
||||
|
||||
const coreLogs = core.peekDirtyState().logs.map((entry) => ({
|
||||
scope: String(entry.scope).toLowerCase(),
|
||||
category: String(entry.category).toLowerCase(),
|
||||
generalId: entry.generalId ?? null,
|
||||
text: entry.text,
|
||||
}));
|
||||
expect(coreLogs).toEqual(projectReferenceLogs(reference));
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -93,6 +93,10 @@ integration('core ↔ legacy command-boundary differential', () => {
|
||||
it.each([
|
||||
['nation declaration', 'fixtures/turn-differential/nation-declaration.json'],
|
||||
['live sortie conquest', 'fixtures/turn-differential/live-sortie-conquest.json'],
|
||||
[
|
||||
'live sortie collapsed nation conflict cleanup',
|
||||
'fixtures/turn-differential/live-sortie-collapse-conflict.json',
|
||||
],
|
||||
['live sortie against a defending general', 'fixtures/turn-differential/live-sortie-defender.json'],
|
||||
[
|
||||
'live sortie against multiple defending generals',
|
||||
@@ -104,6 +108,10 @@ integration('core ↔ legacy command-boundary differential', () => {
|
||||
['live sortie conflict arbitration', 'fixtures/turn-differential/live-sortie-conflict-arbitration.json'],
|
||||
['live sortie tied conflict', 'fixtures/turn-differential/live-sortie-conflict-tie.json'],
|
||||
['live sortie outer lifecycle: conquest', 'fixtures/turn-differential/live-sortie-conquest.json'],
|
||||
[
|
||||
'live sortie outer lifecycle: collapsed nation conflict cleanup',
|
||||
'fixtures/turn-differential/live-sortie-collapse-conflict.json',
|
||||
],
|
||||
['live sortie outer lifecycle: defender', 'fixtures/turn-differential/live-sortie-defender.json'],
|
||||
[
|
||||
'live sortie outer lifecycle: multiple defenders',
|
||||
@@ -173,6 +181,12 @@ integration('core ↔ legacy command-boundary differential', () => {
|
||||
});
|
||||
expect(reference.after.cities.find((city) => city.id === 71)?.supplyState).toBe(1);
|
||||
}
|
||||
if (fixturePath.endsWith('live-sortie-collapse-conflict.json')) {
|
||||
expect(reference.before.cities.find((city) => city.id === 71)?.conflict).toEqual({ 2: 1234 });
|
||||
expect(reference.after.nations.some((nation) => nation.id === 2)).toBe(false);
|
||||
expect(Object.keys(reference.after.cities.find((city) => city.id === 71)?.conflict ?? {})).toEqual([]);
|
||||
expect(Object.keys(core.after.cities.find((city) => city.id === 71)?.conflict ?? {})).toEqual([]);
|
||||
}
|
||||
if (fixturePath.endsWith('live-sortie-conflict-arbitration.json')) {
|
||||
expect(reference.before.cities.find((city) => city.id === 70)?.conflict).toEqual({ 3: 10_000 });
|
||||
expect(reference.after.cities.find((city) => city.id === 70)).toMatchObject({
|
||||
|
||||
@@ -91,6 +91,7 @@ cd "$workspace_root"
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm --filter @sammo-ts/infra prisma:generate
|
||||
pnpm --filter @sammo-ts/common build
|
||||
pnpm --filter @sammo-ts/infra build
|
||||
|
||||
database_url=$(build_database_url "$integration_schema")
|
||||
export POSTGRES_SCHEMA=$integration_schema
|
||||
|
||||
Reference in New Issue
Block a user