591 lines
21 KiB
TypeScript
591 lines
21 KiB
TypeScript
import path from 'node:path';
|
|
import fs from 'node:fs/promises';
|
|
import http from 'node:http';
|
|
import https from 'node:https';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { execFile } from 'node:child_process';
|
|
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
import { createTRPCProxyClient, httpBatchLink } from '@trpc/client';
|
|
import { sealGatewayPassword } from '../src/passwordEnvelope.js';
|
|
|
|
import type { AppRouter as GatewayAppRouter } from '@sammo-ts/gateway-api';
|
|
import type { AppRouter as GameAppRouter } from '@sammo-ts/game-api';
|
|
import { createGatewayApiServer } from '@sammo-ts/gateway-api';
|
|
import { Pm2ProcessManager } from '../../../app/gateway-api/src/orchestrator/pm2ProcessManager.js';
|
|
import {
|
|
createGatewayPostgresConnector,
|
|
createGamePostgresConnector,
|
|
resolvePostgresConfigFromEnv,
|
|
createRedisConnector,
|
|
resolveRedisConfigFromEnv,
|
|
} from '@sammo-ts/infra';
|
|
|
|
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
const workspaceRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
|
|
|
|
const parseEnvFile = (rawText: string): Record<string, string> => {
|
|
const env: Record<string, string> = {};
|
|
const lines = rawText.split(/\r?\n/);
|
|
for (const line of lines) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith('#')) {
|
|
continue;
|
|
}
|
|
const index = trimmed.indexOf('=');
|
|
if (index < 0) {
|
|
continue;
|
|
}
|
|
const key = trimmed.slice(0, index).trim();
|
|
let value = trimmed.slice(index + 1).trim();
|
|
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
|
value = value.slice(1, -1);
|
|
}
|
|
env[key] = value;
|
|
}
|
|
return env;
|
|
};
|
|
|
|
const loadEnv = async () => {
|
|
const envPath = path.join(workspaceRoot, '.env.ci');
|
|
const raw = await fs.readFile(envPath, 'utf8');
|
|
const values = parseEnvFile(raw);
|
|
for (const [key, value] of Object.entries(values)) {
|
|
if (process.env[key] === undefined) {
|
|
process.env[key] = value;
|
|
}
|
|
}
|
|
process.env.INTEGRATION_JOIN_ALLOW_CITY ??= 'true';
|
|
process.env.INTEGRATION_WORLD_SEED ??= 'integration-seed';
|
|
process.env.GATEWAY_WORKSPACE_ROOT ??= workspaceRoot;
|
|
process.env.GATEWAY_WORKTREE_ROOT ??= path.join(workspaceRoot, '.worktrees-test');
|
|
process.env.GATEWAY_DB_SCHEMA ??= 'public';
|
|
process.env.GATEWAY_API_PORT ??= '13000';
|
|
process.env.GAME_API_PORT ??= '14000';
|
|
};
|
|
|
|
const execCommand = (command: string, args: string[], env?: NodeJS.ProcessEnv) =>
|
|
new Promise<void>((resolve, reject) => {
|
|
execFile(command, args, { env, cwd: workspaceRoot }, (error, stdout, stderr) => {
|
|
if (error) {
|
|
reject(new Error(`${command} ${args.join(' ')} failed:\n${stdout}\n${stderr}`));
|
|
return;
|
|
}
|
|
resolve();
|
|
});
|
|
});
|
|
|
|
const ensureSchema = async (schema: string) => {
|
|
const adminUrl = resolvePostgresConfigFromEnv({ schema: 'public' }).url;
|
|
const connector = createGatewayPostgresConnector({ url: adminUrl });
|
|
await connector.connect();
|
|
try {
|
|
await connector.prisma.$executeRawUnsafe(`CREATE SCHEMA IF NOT EXISTS "${schema}"`);
|
|
} finally {
|
|
await connector.disconnect();
|
|
}
|
|
};
|
|
|
|
const truncateSchema = async (schema: string) => {
|
|
const adminUrl = resolvePostgresConfigFromEnv({ schema: 'public' }).url;
|
|
const connector = createGatewayPostgresConnector({ url: adminUrl });
|
|
await connector.connect();
|
|
try {
|
|
const rows = (await connector.prisma.$queryRawUnsafe(
|
|
`SELECT tablename FROM pg_tables WHERE schemaname = '${schema}'`
|
|
)) as Array<{ tablename: string }>;
|
|
if (rows.length === 0) {
|
|
return;
|
|
}
|
|
const tableList = rows.map((row) => `"${schema}"."${row.tablename}"`).join(', ');
|
|
await connector.prisma.$executeRawUnsafe(`TRUNCATE TABLE ${tableList} RESTART IDENTITY CASCADE`);
|
|
} finally {
|
|
await connector.disconnect();
|
|
}
|
|
};
|
|
|
|
const resetDatabase = async (profile: string) => {
|
|
await ensureSchema('public');
|
|
await ensureSchema(profile);
|
|
await execCommand('pnpm', ['--filter', '@sammo-ts/infra', 'prisma:db:push:gateway', '--accept-data-loss'], {
|
|
...process.env,
|
|
POSTGRES_SCHEMA: 'public',
|
|
});
|
|
await execCommand('pnpm', ['--filter', '@sammo-ts/infra', 'prisma:db:push:game', '--accept-data-loss'], {
|
|
...process.env,
|
|
POSTGRES_SCHEMA: profile,
|
|
});
|
|
await truncateSchema('public');
|
|
await truncateSchema(profile);
|
|
};
|
|
|
|
const resetRedis = async () => {
|
|
const redis = createRedisConnector(resolveRedisConfigFromEnv());
|
|
await redis.connect();
|
|
try {
|
|
await redis.client.flushDb();
|
|
} finally {
|
|
await redis.disconnect();
|
|
}
|
|
};
|
|
|
|
const ensureBuildArtifacts = async () => {
|
|
const required = [
|
|
path.join(workspaceRoot, 'app', 'game-api', 'dist', 'index.js'),
|
|
path.join(workspaceRoot, 'app', 'game-engine', 'dist', 'index.js'),
|
|
];
|
|
for (const filePath of required) {
|
|
try {
|
|
await fs.access(filePath);
|
|
} catch {
|
|
throw new Error(`Missing build artifact: ${filePath}. Run pnpm build first.`);
|
|
}
|
|
}
|
|
};
|
|
|
|
const createGatewayClient = (baseUrl: string, trpcPath: string, sessionTokenRef: { value?: string }) =>
|
|
createTRPCProxyClient<GatewayAppRouter>({
|
|
links: [
|
|
httpBatchLink({
|
|
url: `${baseUrl}${trpcPath}`,
|
|
headers: () =>
|
|
sessionTokenRef.value
|
|
? {
|
|
'x-session-token': sessionTokenRef.value,
|
|
}
|
|
: {},
|
|
}),
|
|
],
|
|
});
|
|
|
|
const createGameClient = (baseUrl: string, trpcPath: string, accessTokenRef: { value?: string }) =>
|
|
createTRPCProxyClient<GameAppRouter>({
|
|
links: [
|
|
httpBatchLink({
|
|
url: `${baseUrl}${trpcPath}`,
|
|
headers: () =>
|
|
accessTokenRef.value
|
|
? {
|
|
authorization: `Bearer ${accessTokenRef.value}`,
|
|
}
|
|
: {},
|
|
}),
|
|
],
|
|
});
|
|
|
|
const buildProcessName = (profileName: string, role: 'api' | 'daemon' | 'tournament'): string =>
|
|
`sammo:${profileName}:${role === 'api' ? 'game-api' : role === 'daemon' ? 'turn-daemon' : 'tournament-worker'}`;
|
|
|
|
const waitForPm2Online = async (manager: Pm2ProcessManager, names: string[], timeoutMs = 30_000) => {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
const list = await manager.list();
|
|
const statusMap = new Map(list.map((process) => [process.name, process.status]));
|
|
if (names.every((name) => statusMap.get(name) === 'online')) {
|
|
return;
|
|
}
|
|
await sleep(500);
|
|
}
|
|
throw new Error(`PM2 processes did not reach online: ${names.join(', ')}`);
|
|
};
|
|
|
|
const cleanupPm2 = async (manager: Pm2ProcessManager, names: string[]) => {
|
|
for (const name of names) {
|
|
try {
|
|
await manager.stop(name);
|
|
} catch {
|
|
// Cleanup is best-effort when the process is already stopped.
|
|
}
|
|
try {
|
|
await manager.delete(name);
|
|
} catch {
|
|
// Cleanup is best-effort when the process is already deleted.
|
|
}
|
|
}
|
|
};
|
|
|
|
const waitForTurnDaemonStatus = async (gameClient: ReturnType<typeof createGameClient>, timeoutMs = 90_000) => {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let lastError: string | null = null;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
const status = await gameClient.turnDaemon.status.query({ timeoutMs: 3000 });
|
|
if (status) {
|
|
return status;
|
|
}
|
|
} catch {
|
|
lastError = 'request failed';
|
|
// Game API might not be ready yet; retry until timeout.
|
|
}
|
|
await sleep(200);
|
|
}
|
|
throw new Error(`turn daemon status timeout${lastError ? ` (${lastError})` : ''}`);
|
|
};
|
|
|
|
const waitForHttpOk = async (url: string, timeoutMs = 15_000) => {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
const parsedUrl = new URL(url);
|
|
const client = parsedUrl.protocol === 'https:' ? https : http;
|
|
const status = await new Promise<number>((resolve, reject) => {
|
|
const request = client.request(parsedUrl, (response) => {
|
|
response.resume();
|
|
resolve(response.statusCode ?? 0);
|
|
});
|
|
request.on('error', reject);
|
|
request.end();
|
|
});
|
|
if (status >= 200 && status < 300) {
|
|
return;
|
|
}
|
|
} catch {
|
|
// The service can refuse connections while PM2 is still starting it.
|
|
}
|
|
await sleep(250);
|
|
}
|
|
throw new Error(`health check timeout: ${url}`);
|
|
};
|
|
|
|
const runTurn = async (gameClient: ReturnType<typeof createGameClient>) => {
|
|
const status = await waitForTurnDaemonStatus(gameClient);
|
|
const prevRunAt = status.lastRunAt;
|
|
const targetTime = status.nextTurnTime
|
|
? new Date(new Date(status.nextTurnTime).getTime() + 5 * 60_000).toISOString()
|
|
: new Date(Date.now() + 5 * 60_000).toISOString();
|
|
await gameClient.turnDaemon.run.mutate({
|
|
reason: 'manual',
|
|
targetTime,
|
|
});
|
|
const deadline = Date.now() + 20_000;
|
|
while (Date.now() < deadline) {
|
|
const nextStatus = await waitForTurnDaemonStatus(gameClient);
|
|
if (!nextStatus.running && nextStatus.lastRunAt && nextStatus.lastRunAt !== prevRunAt) {
|
|
return nextStatus;
|
|
}
|
|
await sleep(200);
|
|
}
|
|
throw new Error('turn run timeout');
|
|
};
|
|
|
|
const parseSseFrame = (raw: string): { event?: string; data?: string } | null => {
|
|
if (!raw.trim()) {
|
|
return null;
|
|
}
|
|
const lines = raw.split(/\r?\n/);
|
|
let event: string | undefined;
|
|
const dataLines: string[] = [];
|
|
for (const line of lines) {
|
|
if (line.startsWith('event:')) {
|
|
event = line.slice(6).trim();
|
|
continue;
|
|
}
|
|
if (line.startsWith('data:')) {
|
|
dataLines.push(line.slice(5).trimStart());
|
|
}
|
|
}
|
|
if (!event && dataLines.length === 0) {
|
|
return null;
|
|
}
|
|
return { event, data: dataLines.join('\n') };
|
|
};
|
|
|
|
const waitForSseEvent = async (options: {
|
|
url: string;
|
|
token: string;
|
|
eventName: string;
|
|
timeoutMs?: number;
|
|
minAtMs?: number;
|
|
}) =>
|
|
new Promise<Record<string, unknown>>((resolve, reject) => {
|
|
const parsedUrl = new URL(options.url);
|
|
const client = parsedUrl.protocol === 'https:' ? https : http;
|
|
let settled = false;
|
|
let timer: NodeJS.Timeout | null = null;
|
|
const request = client.request(
|
|
parsedUrl,
|
|
{
|
|
headers: {
|
|
authorization: `Bearer ${options.token}`,
|
|
},
|
|
},
|
|
(response) => {
|
|
if (response.statusCode && response.statusCode >= 400) {
|
|
cleanup();
|
|
settled = true;
|
|
reject(new Error(`SSE request failed with ${response.statusCode}`));
|
|
response.resume();
|
|
return;
|
|
}
|
|
response.setEncoding('utf8');
|
|
let buffer = '';
|
|
const onData = (chunk: string) => {
|
|
buffer += chunk;
|
|
let index = buffer.indexOf('\n\n');
|
|
while (index >= 0) {
|
|
const rawFrame = buffer.slice(0, index);
|
|
buffer = buffer.slice(index + 2);
|
|
const frame = parseSseFrame(rawFrame);
|
|
if (!frame || frame.event !== options.eventName || !frame.data) {
|
|
index = buffer.indexOf('\n\n');
|
|
continue;
|
|
}
|
|
try {
|
|
const parsed = JSON.parse(frame.data) as Record<string, unknown>;
|
|
if (typeof options.minAtMs === 'number') {
|
|
const at = typeof parsed.at === 'string' ? new Date(parsed.at).getTime() : NaN;
|
|
if (Number.isFinite(at) && at < options.minAtMs) {
|
|
index = buffer.indexOf('\n\n');
|
|
continue;
|
|
}
|
|
}
|
|
cleanup();
|
|
settled = true;
|
|
resolve(parsed);
|
|
return;
|
|
} catch (error) {
|
|
cleanup();
|
|
settled = true;
|
|
reject(error);
|
|
return;
|
|
}
|
|
}
|
|
};
|
|
response.on('data', onData);
|
|
response.on('end', () => {
|
|
if (settled) {
|
|
return;
|
|
}
|
|
cleanup();
|
|
settled = true;
|
|
reject(new Error('SSE connection closed before event'));
|
|
});
|
|
}
|
|
);
|
|
|
|
const cleanup = () => {
|
|
if (timer) {
|
|
clearTimeout(timer);
|
|
}
|
|
request.destroy();
|
|
};
|
|
|
|
request.on('error', (error) => {
|
|
if (settled) {
|
|
return;
|
|
}
|
|
cleanup();
|
|
settled = true;
|
|
reject(error);
|
|
});
|
|
request.end();
|
|
|
|
timer = setTimeout(() => {
|
|
if (settled) {
|
|
return;
|
|
}
|
|
cleanup();
|
|
settled = true;
|
|
reject(new Error('SSE timeout'));
|
|
}, options.timeoutMs ?? 30_000);
|
|
});
|
|
|
|
describe('pm2 orchestrator e2e', () => {
|
|
let gatewayServer: Awaited<ReturnType<typeof createGatewayApiServer>> | null = null;
|
|
let pm2Manager: Pm2ProcessManager | null = null;
|
|
let profile = 'che';
|
|
let scenario = '2';
|
|
let scenarioId = 2;
|
|
let profileName = 'che:2';
|
|
let apiPort = 14000;
|
|
let processNames: string[] = [];
|
|
|
|
beforeAll(async () => {
|
|
await loadEnv();
|
|
process.chdir(workspaceRoot);
|
|
await ensureBuildArtifacts();
|
|
|
|
profile = process.env.PROFILE ?? 'che';
|
|
scenario = process.env.SCENARIO ?? '2';
|
|
scenarioId = Number(scenario);
|
|
if (!Number.isFinite(scenarioId)) {
|
|
throw new Error(`SCENARIO must be numeric, got ${scenario}`);
|
|
}
|
|
profileName = `${profile}:${scenario}`;
|
|
apiPort = Number(process.env.GAME_API_PORT ?? '14000');
|
|
processNames = [
|
|
buildProcessName(profileName, 'api'),
|
|
buildProcessName(profileName, 'daemon'),
|
|
buildProcessName(profileName, 'tournament'),
|
|
];
|
|
|
|
await resetDatabase(profile);
|
|
await resetRedis();
|
|
|
|
pm2Manager = new Pm2ProcessManager();
|
|
await cleanupPm2(pm2Manager, processNames);
|
|
|
|
gatewayServer = await createGatewayApiServer();
|
|
await gatewayServer.app.listen({
|
|
host: gatewayServer.config.host,
|
|
port: gatewayServer.config.port,
|
|
});
|
|
}, 60_000);
|
|
|
|
afterAll(async () => {
|
|
if (pm2Manager) {
|
|
await cleanupPm2(pm2Manager, processNames);
|
|
}
|
|
if (gatewayServer) {
|
|
await gatewayServer.app.close();
|
|
}
|
|
}, 30_000);
|
|
|
|
it('starts game-api/turn-daemon/tournament-worker via PM2 and serves commands', async () => {
|
|
if (!gatewayServer || !pm2Manager) {
|
|
throw new Error('test setup failed');
|
|
}
|
|
const bootstrapToken = process.env.GATEWAY_BOOTSTRAP_TOKEN ?? '';
|
|
if (!bootstrapToken) {
|
|
throw new Error('GATEWAY_BOOTSTRAP_TOKEN is required for bootstrapLocal');
|
|
}
|
|
|
|
const gatewayUrl = `http://localhost:${gatewayServer.config.port}`;
|
|
const gameUrl = `http://localhost:${apiPort}`;
|
|
const adminSessionRef: { value?: string } = {};
|
|
const adminGameAccessRef: { value?: string } = {};
|
|
const accessTokenRef: { value?: string } = {};
|
|
const gatewayClient = createGatewayClient(gatewayUrl, gatewayServer.config.trpcPath, adminSessionRef);
|
|
const gameTrpcPath = process.env.GAME_TRPC_PATH ?? process.env.TRPC_PATH ?? '/trpc';
|
|
const gameClientPublic = createGameClient(gameUrl, gameTrpcPath, { value: undefined });
|
|
const gameClientAdmin = createGameClient(gameUrl, gameTrpcPath, adminGameAccessRef);
|
|
const gameClientAuthed = createGameClient(gameUrl, gameTrpcPath, accessTokenRef);
|
|
|
|
const bootstrap = await gatewayClient.auth.bootstrapLocal.mutate({
|
|
token: bootstrapToken,
|
|
username: 'admin',
|
|
password: 'admin-pass-123',
|
|
displayName: 'Admin',
|
|
});
|
|
adminSessionRef.value = bootstrap.sessionToken;
|
|
expect(bootstrap.user.username).toBe('admin');
|
|
|
|
await gatewayClient.admin.users.createLocal.mutate({
|
|
username: 'e2e-user',
|
|
password: 'e2e-pass-123',
|
|
displayName: 'IntegrationUser',
|
|
});
|
|
|
|
await gatewayClient.admin.profiles.upsert.mutate({
|
|
profile,
|
|
scenario,
|
|
apiPort,
|
|
status: 'RUNNING',
|
|
});
|
|
|
|
await gatewayClient.admin.profiles.installNow.mutate({
|
|
profileName,
|
|
install: {
|
|
scenarioId,
|
|
turnTermMinutes: 1,
|
|
sync: false,
|
|
fiction: 0,
|
|
extend: true,
|
|
blockGeneralCreate: 0,
|
|
npcMode: 0,
|
|
showImgLevel: 0,
|
|
tournamentTrig: false,
|
|
joinMode: 'full',
|
|
autorunUser: null,
|
|
},
|
|
});
|
|
|
|
await gatewayClient.admin.profiles.reconcileNow.mutate();
|
|
|
|
await waitForPm2Online(pm2Manager, processNames);
|
|
|
|
await waitForHttpOk(`${gameUrl}/healthz`);
|
|
|
|
const adminGameSession = await gatewayClient.auth.issueGameSession.mutate({
|
|
sessionToken: adminSessionRef.value ?? '',
|
|
profile: profileName,
|
|
});
|
|
const adminAccess = await gameClientPublic.auth.exchangeGatewayToken.mutate({
|
|
gatewayToken: adminGameSession.gameToken,
|
|
});
|
|
adminGameAccessRef.value = adminAccess.accessToken;
|
|
|
|
await waitForTurnDaemonStatus(gameClientAdmin);
|
|
|
|
const login = await gatewayClient.auth.login.mutate({
|
|
username: 'e2e-user',
|
|
credential: sealGatewayPassword('e2e-pass-123', await gatewayClient.auth.passwordKey.query()),
|
|
});
|
|
|
|
const gameSession = await gatewayClient.auth.issueGameSession.mutate({
|
|
sessionToken: login.sessionToken,
|
|
profile: profileName,
|
|
});
|
|
|
|
const access = await gameClientPublic.auth.exchangeGatewayToken.mutate({
|
|
gatewayToken: gameSession.gameToken,
|
|
});
|
|
accessTokenRef.value = access.accessToken;
|
|
|
|
const publicMap = await gameClientPublic.public.getCachedMap.query();
|
|
expect(publicMap.result).toBe(true);
|
|
const cityCandidates = publicMap.cityList
|
|
.map((row: number[]) => ({
|
|
id: row[0],
|
|
level: row[1],
|
|
nationId: row[3],
|
|
}))
|
|
.filter(
|
|
(city: { id: number; level: number; nationId: number }) =>
|
|
(city.level === 5 || city.level === 6) && city.nationId === 0
|
|
)
|
|
.map((city: { id: number }) => city.id);
|
|
expect(cityCandidates.length).toBeGreaterThan(0);
|
|
|
|
const createdGeneral = await gameClientAuthed.join.createGeneral.mutate({
|
|
name: 'IntegrationUser',
|
|
leadership: 55,
|
|
strength: 55,
|
|
intel: 55,
|
|
character: 'Random',
|
|
});
|
|
expect(createdGeneral.generalId).toBeGreaterThan(0);
|
|
|
|
const eventsPath = process.env.GAME_API_EVENTS_PATH ?? '/events';
|
|
const runStartMs = Date.now();
|
|
const ssePromise = waitForSseEvent({
|
|
url: `${gameUrl}${eventsPath}`,
|
|
token: accessTokenRef.value ?? '',
|
|
eventName: 'turnCompleted',
|
|
minAtMs: runStartMs,
|
|
timeoutMs: 30_000,
|
|
});
|
|
|
|
const runStatus = await runTurn(gameClientAdmin);
|
|
expect(runStatus.lastRunAt).toBeTruthy();
|
|
|
|
const realtimeEvent = await ssePromise;
|
|
expect(realtimeEvent.type).toBe('turnCompleted');
|
|
|
|
const gameDatabaseUrl = resolvePostgresConfigFromEnv({ schema: profile }).url;
|
|
const connector = createGamePostgresConnector({ url: gameDatabaseUrl });
|
|
await connector.connect();
|
|
try {
|
|
const worldState = await connector.prisma.worldState.findFirst({
|
|
select: { updatedAt: true },
|
|
});
|
|
expect(worldState?.updatedAt).toBeTruthy();
|
|
if (worldState?.updatedAt) {
|
|
expect(worldState.updatedAt.getTime()).toBeGreaterThanOrEqual(runStartMs);
|
|
}
|
|
} finally {
|
|
await connector.disconnect();
|
|
}
|
|
}, 120_000);
|
|
});
|