feat: add durable release controller and admin deploy modes

This commit is contained in:
2026-08-01 03:27:38 +00:00
parent 395b60cdbe
commit 7b466de0a6
37 changed files with 2778 additions and 35 deletions
+159
View File
@@ -39,6 +39,7 @@ const ROLE_SUPERUSER = 'superuser';
const ROLE_ADMIN_USERS = 'admin.users.manage';
const ROLE_ADMIN_USERS_CREATE = 'admin.users.create';
const ROLE_ADMIN_PROFILES = 'admin.profiles.manage';
const ROLE_ADMIN_RELEASES = 'admin.releases.manage';
const ROLE_ADMIN_NOTICE = 'admin.notice.manage';
const ROLE_RESET_SCHEDULE = 'admin.reset.schedule';
const ROLE_RESUME_WHEN_STOPPED = 'admin.resume.when-stopped';
@@ -242,6 +243,12 @@ const profileAdminProcedure = adminProcedure.use(({ ctx, next }) => {
return next();
});
const releaseAdminProcedure = adminProcedure.use(({ ctx, next }) => {
const adminAuth = requireAdminAuth(ctx);
assertPermission(adminAuth, ROLE_ADMIN_RELEASES);
return next();
});
const zUserLookupInput = z
.object({
id: z.string().min(1).optional(),
@@ -766,6 +773,60 @@ export const adminRouter = router({
});
}
}),
requestDeploy: adminProcedure
.input(
z.object({
profileName: z.string().min(1),
sourceMode: zSourceMode,
sourceRef: z.string().min(1).max(128),
reason: z.string().max(200).optional(),
})
)
.mutation(async ({ ctx, input }) => {
const adminAuth = requireAdminAuth(ctx);
assertPermission(adminAuth, ROLE_ADMIN_PROFILES, input.profileName);
const profile = await ctx.profiles.getProfile(input.profileName);
if (!profile) {
throw new TRPCError({ code: 'NOT_FOUND', message: 'Profile not found.' });
}
let sourceRef = input.sourceRef.trim();
try {
const resolved =
input.sourceMode === 'BRANCH'
? await resolveGitBranchCommitSha(sourceRef)
: await resolveGitCommitSha(sourceRef);
if (input.sourceMode === 'COMMIT') {
sourceRef = resolved;
}
const scenarios = await listScenarioPreviews({ gitRef: resolved });
if (!scenarios.some((scenario) => String(scenario.id) === profile.scenario)) {
throw new Error('Current scenario is not available at source.');
}
} catch {
throw new TRPCError({
code: 'BAD_REQUEST',
message: 'Source is invalid or does not contain the current scenario.',
});
}
try {
return await ctx.profiles.createOperation({
profileName: input.profileName,
type: 'DEPLOY',
sourceMode: input.sourceMode,
sourceRef,
reason: input.reason,
requestedBy: adminAuth.user.id,
});
} catch (error) {
if (!isUniqueConstraintError(error)) {
throw error;
}
throw new TRPCError({
code: 'CONFLICT',
message: 'This profile already has a queued or running operation.',
});
}
}),
requestRuntime: adminProcedure
.input(
z.object({
@@ -845,6 +906,103 @@ export const adminRouter = router({
}
}),
}),
releases: router({
gatewayState: releaseAdminProcedure.query(({ ctx }) => ctx.releases.getState()),
list: releaseAdminProcedure
.input(z.object({ limit: z.number().int().min(1).max(200).optional() }).optional())
.query(({ ctx, input }) => ctx.releases.listOperations(input?.limit)),
requestGatewayDeploy: releaseAdminProcedure
.input(
z.object({
sourceMode: zSourceMode,
sourceRef: z.string().min(1).max(128),
reason: z.string().max(200).optional(),
})
)
.mutation(async ({ ctx, input }) => {
const adminAuth = requireAdminAuth(ctx);
let sourceRef = input.sourceRef.trim();
try {
const resolved =
input.sourceMode === 'BRANCH'
? await resolveGitBranchCommitSha(sourceRef)
: await resolveGitCommitSha(sourceRef);
if (input.sourceMode === 'COMMIT') {
sourceRef = resolved;
}
} catch {
throw new TRPCError({ code: 'BAD_REQUEST', message: 'Gateway release source is invalid.' });
}
try {
return await ctx.releases.createOperation({
type: 'DEPLOY',
sourceMode: input.sourceMode,
sourceRef,
reason: input.reason,
requestedBy: adminAuth.user.id,
});
} catch (error) {
if (!isUniqueConstraintError(error)) {
throw error;
}
throw new TRPCError({ code: 'CONFLICT', message: 'A gateway release is already active.' });
}
}),
requestGatewayRollback: releaseAdminProcedure
.input(z.object({ reason: z.string().max(200).optional() }).optional())
.mutation(async ({ ctx, input }) => {
const adminAuth = requireAdminAuth(ctx);
const state = await ctx.releases.getState();
if (!state.previousCommitSha || !state.previousWorkspace) {
throw new TRPCError({ code: 'BAD_REQUEST', message: 'No previous gateway release is available.' });
}
try {
return await ctx.releases.createOperation({
type: 'ROLLBACK',
sourceMode: 'COMMIT',
sourceRef: state.previousCommitSha,
payload: {
expectedWorkspace: state.previousWorkspace,
replacedCommitSha: state.activeCommitSha ?? null,
},
reason: input?.reason,
requestedBy: adminAuth.user.id,
});
} catch (error) {
if (!isUniqueConstraintError(error)) {
throw error;
}
throw new TRPCError({ code: 'CONFLICT', message: 'A gateway release is already active.' });
}
}),
cancel: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => {
if (!(await ctx.releases.cancelOperation(input.id))) {
throw new TRPCError({ code: 'CONFLICT', message: 'Only queued releases can be cancelled.' });
}
return { ok: true };
}),
retry: releaseAdminProcedure.input(z.object({ id: z.string().uuid() })).mutation(async ({ ctx, input }) => {
const adminAuth = requireAdminAuth(ctx);
try {
const operation = await ctx.releases.retryOperation(input.id, adminAuth.user.id);
if (!operation) {
throw new TRPCError({
code: 'CONFLICT',
message: 'Only failed or cancelled releases can be retried.',
});
}
return operation;
} catch (error) {
if (error instanceof TRPCError) {
throw error;
}
if (!isUniqueConstraintError(error)) {
throw error;
}
throw new TRPCError({ code: 'CONFLICT', message: 'A gateway release is already active.' });
}
}),
}),
profiles: router({
list: adminProcedure.query(async ({ ctx }) => {
const profiles = await ctx.profiles.listProfiles();
@@ -883,6 +1041,7 @@ export const adminRouter = router({
activeOperation: activeOperationByProfile.get(profile.profileName) ?? null,
runtime: runtimeMap.get(profile.profileName) ?? {
profileName: profile.profileName,
frontendRunning: false,
apiRunning: false,
daemonRunning: false,
auctionRunning: false,
+7
View File
@@ -4,6 +4,10 @@ import type { UserRepository } from './auth/userRepository.js';
import type { KakaoOAuthClient } from './auth/kakaoClient.js';
import type { OAuthSessionStore } from './auth/oauthSessionStore.js';
import type { GatewayProfileRepository } from './orchestrator/profileRepository.js';
import {
createGatewayReleaseRepository,
type GatewayReleaseRepository,
} from './orchestrator/gatewayReleaseRepository.js';
import type { GatewayOrchestratorHandle } from './orchestrator/gatewayOrchestrator.js';
import type { GatewayProfileStatusService } from './lobby/profileStatusService.js';
import type { GatewayPrismaClient } from '@sammo-ts/infra';
@@ -26,6 +30,7 @@ export interface GatewayApiContext {
localAccountGraceDays: number;
passwordEnvelope: PasswordEnvelopeService;
profiles: GatewayProfileRepository;
releases: GatewayReleaseRepository;
orchestrator: GatewayOrchestratorHandle;
profileStatus: GatewayProfileStatusService;
requestHeaders: Record<string, string | string[] | undefined>;
@@ -49,6 +54,7 @@ export const createGatewayApiContext = (options: {
localAccountGraceDays: number;
passwordEnvelope: PasswordEnvelopeService;
profiles: GatewayProfileRepository;
releases?: GatewayReleaseRepository;
orchestrator: GatewayOrchestratorHandle;
profileStatus: GatewayProfileStatusService;
requestHeaders?: Record<string, string | string[] | undefined>;
@@ -69,6 +75,7 @@ export const createGatewayApiContext = (options: {
localAccountGraceDays: options.localAccountGraceDays,
passwordEnvelope: options.passwordEnvelope,
profiles: options.profiles,
releases: options.releases ?? createGatewayReleaseRepository(options.prisma),
orchestrator: options.orchestrator,
profileStatus: options.profileStatus,
requestHeaders: options.requestHeaders ?? {},
+6
View File
@@ -14,7 +14,13 @@ export { GatewayPrisma };
export type JsonObject = GatewayPrisma.JsonObject;
export type JsonArray = GatewayPrisma.JsonArray;
export * from './orchestrator/profileRepository.js';
export * from './orchestrator/gatewayReleaseRepository.js';
export * from './orchestrator/gatewayOrchestrator.js';
export * from './orchestrator/workspaceManager.js';
export * from './orchestrator/buildRunner.js';
export * from './orchestrator/processManager.js';
export * from './orchestrator/pm2ProcessManager.js';
export * from './orchestrator/releaseManifest.js';
export * from './auth/userRepository.js';
export * from './auth/passwordHasher.js';
export * from './auth/inMemoryUserRepository.js';
@@ -18,6 +18,7 @@ import type {
} from './profileRepository.js';
import type { GitWorkspaceManager } from './workspaceManager.js';
import type { AdminSeedUser } from './seedProfileDatabase.js';
import { assertReleaseComponents, readReleaseManifest } from './releaseManifest.js';
export interface GatewayProcessConfig {
workspaceRoot: string;
@@ -37,10 +38,13 @@ export interface GatewayOrchestratorOptions {
scheduleIntervalMs: number;
buildIntervalMs: number;
adminActionIntervalMs: number;
profileReadinessTimeoutMs?: number;
now?: () => Date;
fetchImpl?: typeof fetch;
}
export interface ProfileRuntimeState {
frontendRunning: boolean;
apiRunning: boolean;
daemonRunning: boolean;
auctionRunning: boolean;
@@ -73,6 +77,7 @@ export const planProfileReconcile = (
if (status === 'RUNNING' || status === 'PREOPEN' || status === 'PAUSED' || status === 'COMPLETED') {
return {
shouldStart: !(
runtime.frontendRunning &&
runtime.apiRunning &&
runtime.daemonRunning &&
runtime.auctionRunning &&
@@ -85,6 +90,7 @@ export const planProfileReconcile = (
return {
shouldStart: false,
shouldStop:
runtime.frontendRunning ||
runtime.apiRunning ||
runtime.daemonRunning ||
runtime.auctionRunning ||
@@ -301,18 +307,20 @@ const parseInstallOptions = (
const buildProcessName = (
profileName: string,
role: 'api' | 'daemon' | 'auction' | 'battle-sim' | 'tournament'
role: 'frontend' | 'api' | 'daemon' | 'auction' | 'battle-sim' | 'tournament'
): string =>
`sammo:${profileName}:${
role === 'api'
? 'game-api'
: role === 'daemon'
? 'turn-daemon'
: role === 'auction'
? 'auction-worker'
: role === 'battle-sim'
? 'battle-sim-worker'
: 'tournament-worker'
role === 'frontend'
? 'game-frontend'
: role === 'api'
? 'game-api'
: role === 'daemon'
? 'turn-daemon'
: role === 'auction'
? 'auction-worker'
: role === 'battle-sim'
? 'battle-sim-worker'
: 'tournament-worker'
}`;
const isMissingProcessError = (error: unknown): boolean =>
@@ -335,6 +343,7 @@ export const buildProcessDefinitions = (
profile: GatewayProfileRecord,
config: GatewayProcessConfig
): {
frontend: { name: string; script: string; cwd: string; args: string[]; env: Record<string, string> };
api: { name: string; script: string; cwd: string; env: Record<string, string> };
daemon: { name: string; script: string; cwd: string; env: Record<string, string> };
auction: { name: string; script: string; cwd: string; env: Record<string, string> };
@@ -342,12 +351,16 @@ export const buildProcessDefinitions = (
tournament: { name: string; script: string; cwd: string; env: Record<string, string> };
} => {
const baseEnv = { ...(config.baseEnv ?? {}) };
const frontendName = buildProcessName(profile.profileName, 'frontend');
const apiName = buildProcessName(profile.profileName, 'api');
const daemonName = buildProcessName(profile.profileName, 'daemon');
const auctionName = buildProcessName(profile.profileName, 'auction');
const battleSimName = buildProcessName(profile.profileName, 'battle-sim');
const tournamentName = buildProcessName(profile.profileName, 'tournament');
const runtimeWorkspace = profile.buildWorkspace ?? config.workspaceRoot;
const frontendCwd = path.join(runtimeWorkspace, 'app', 'game-frontend');
const frontendOutDir = buildProfileFrontendOutDir(runtimeWorkspace, profile.profileName);
const frontendScript = path.join(runtimeWorkspace, 'node_modules', 'vite', 'bin', 'vite.js');
const apiCwd = path.join(runtimeWorkspace, 'app', 'game-api');
const daemonCwd = path.join(runtimeWorkspace, 'app', 'game-engine');
const apiScript = path.join(apiCwd, 'dist', 'index.js');
@@ -373,6 +386,13 @@ export const buildProcessDefinitions = (
TURN_PROFILE_NAME: profile.profileName,
};
return {
frontend: {
name: frontendName,
script: frontendScript,
cwd: frontendCwd,
args: ['preview', '--host', '0.0.0.0', '--port', String(profile.apiPort - 1), '--outDir', frontendOutDir],
env: baseEnv,
},
api: {
name: apiName,
script: apiScript,
@@ -415,6 +435,39 @@ export const buildProcessDefinitions = (
};
};
const sanitizeArtifactName = (value: string): string => value.replace(/[^0-9A-Za-z._-]+/g, '_');
export const buildProfileFrontendOutDir = (workspaceRoot: string, profileName: string): string =>
path.join(workspaceRoot, '.release-dist', sanitizeArtifactName(profileName), 'game-frontend');
export const buildProfileFrontendCommands = (
workspaceRoot: string,
profile: Pick<GatewayProfileRecord, 'profileName' | 'profile' | 'apiPort'>,
env?: Record<string, string>
): BuildCommand[] => {
const buildEnv = {
...(env ?? {}),
VITE_APP_BASE_PATH: `/${profile.profile}`,
VITE_GAME_API_URL: `/${profile.profile}/api/trpc`,
VITE_GAME_SSE_URL: `/${profile.profile}/api/events`,
};
const outDir = buildProfileFrontendOutDir(workspaceRoot, profile.profileName);
return [
{
command: 'pnpm',
args: ['--filter', '@sammo-ts/game-frontend', 'exec', 'vue-tsc', '--noEmit'],
cwd: workspaceRoot,
env: buildEnv,
},
{
command: 'pnpm',
args: ['--filter', '@sammo-ts/game-frontend', 'exec', 'vite', 'build', '--outDir', outDir],
cwd: workspaceRoot,
env: buildEnv,
},
];
};
export const buildWorkspaceCommands = (
workspaceRoot: string,
needsInstall: boolean,
@@ -424,7 +477,7 @@ export const buildWorkspaceCommands = (
if (needsInstall) {
commands.push({
command: 'pnpm',
args: ['install'],
args: ['install', '--frozen-lockfile'],
cwd: workspaceRoot,
env,
});
@@ -462,6 +515,7 @@ export const buildProfileMigrationCommand = (
const mapRuntimeStates = (profileNames: string[], processNames: Map<string, boolean>): ProfileRuntimeSnapshot[] =>
profileNames.map((profileName) => {
const frontendName = buildProcessName(profileName, 'frontend');
const apiName = buildProcessName(profileName, 'api');
const daemonName = buildProcessName(profileName, 'daemon');
const auctionName = buildProcessName(profileName, 'auction');
@@ -469,6 +523,7 @@ const mapRuntimeStates = (profileNames: string[], processNames: Map<string, bool
const tournamentName = buildProcessName(profileName, 'tournament');
return {
profileName,
frontendRunning: processNames.get(frontendName) ?? false,
apiRunning: processNames.get(apiName) ?? false,
daemonRunning: processNames.get(daemonName) ?? false,
auctionRunning: processNames.get(auctionName) ?? false,
@@ -487,7 +542,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
private readonly scheduleIntervalMs: number;
private readonly buildIntervalMs: number;
private readonly adminActionIntervalMs: number;
private readonly profileReadinessTimeoutMs: number;
private readonly now: () => Date;
private readonly fetchImpl: typeof fetch;
private reconcileTimer?: NodeJS.Timeout;
private scheduleTimer?: NodeJS.Timeout;
private buildTimer?: NodeJS.Timeout;
@@ -513,7 +570,9 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
this.scheduleIntervalMs = options.scheduleIntervalMs;
this.buildIntervalMs = options.buildIntervalMs;
this.adminActionIntervalMs = options.adminActionIntervalMs;
this.profileReadinessTimeoutMs = options.profileReadinessTimeoutMs ?? 30_000;
this.now = options.now ?? (() => new Date());
this.fetchImpl = options.fetchImpl ?? fetch;
}
start(): void {
@@ -668,7 +727,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
startedAt,
error: null,
});
const { result, workspace } = await this.runBuildCommands(queued.buildCommitSha);
const { result, workspace } = await this.runBuildCommands(queued.buildCommitSha, queued);
const completedAt = this.now().toISOString();
if (result.ok) {
await this.repository.updateWorkspaceUsage(
@@ -863,6 +922,19 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
}
}
await assertLease();
if (operation.type === 'DEPLOY') {
const result = await this.handleProfileDeploy(profile, commitSha, assertLease, operation.id);
if (!result.ok) {
throw new Error(result.detail);
}
await this.repository.completeOperation(
operation.id,
'SUCCEEDED',
{ resolvedCommitSha: commitSha, error: null },
this.operationLeaseOwner
);
return;
}
const payload = normalizeMeta(operation.payload);
const install = isRecord(payload.install) ? payload.install : {};
const installOperationId =
@@ -923,6 +995,149 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
}
}
private async handleProfileDeploy(
profile: GatewayProfileRecord,
commitSha: string,
assertLease: () => Promise<void>,
operationId: string
): Promise<{ ok: true } | { ok: false; detail: string }> {
if (this.buildInFlight) {
return { ok: false, detail: 'build already in progress' };
}
this.buildInFlight = true;
const shouldRun = ['RUNNING', 'PREOPEN', 'PAUSED', 'COMPLETED'].includes(profile.status);
const updateClaimedProfile = async (patch: GatewayClaimedProfileUpdate): Promise<GatewayProfileRecord> => {
if (!this.repository.updateProfileForOperation) {
throw new Error('Profile deploy requires lease-fenced profile updates.');
}
const updated = await this.repository.updateProfileForOperation(
operationId,
this.operationLeaseOwner,
profile.profileName,
patch
);
if (!updated) {
throw new OperationLeaseLostError(`Operation lease lost while deploying profile: ${operationId}`);
}
return updated;
};
let oldRuntimeStopped = false;
try {
const startedAt = this.now().toISOString();
await updateClaimedProfile({
buildStatus: 'RUNNING',
buildRequestedAt: startedAt,
buildStartedAt: startedAt,
buildError: null,
});
const workspace = await this.workspaceManager.prepare(commitSha);
const manifest = await readReleaseManifest(workspace.root);
assertReleaseComponents(manifest, ['game-api', 'game-engine', 'game-frontend']);
const commands = [
...buildWorkspaceCommands(workspace.root, workspace.needsInstall, this.processConfig.baseEnv),
...buildProfileFrontendCommands(workspace.root, profile, this.processConfig.baseEnv),
];
const result = await this.buildRunner.run(commands);
await assertLease();
if (!result.ok) {
const detail = result.output.slice(-4000) || 'selected workspace build failed';
await updateClaimedProfile({
buildStatus: 'FAILED',
buildCompletedAt: this.now().toISOString(),
buildError: detail,
});
return { ok: false, detail };
}
await this.stopProfile(profile, assertLease);
oldRuntimeStopped = true;
const profileDatabaseUrl = this.resolveProfileDatabaseUrl(profile);
const migration = await this.runProfileMigration(workspace.root, profileDatabaseUrl);
await assertLease();
if (!migration.ok) {
const detail = migration.output.slice(-4000) || 'profile database migration failed';
if (shouldRun) {
await this.startProfile(profile, assertLease);
oldRuntimeStopped = false;
}
await updateClaimedProfile({
buildStatus: 'FAILED',
buildCompletedAt: this.now().toISOString(),
buildError: detail,
});
return { ok: false, detail };
}
const completedAt = this.now().toISOString();
const candidate: GatewayProfileRecord = {
...profile,
buildStatus: 'SUCCEEDED',
buildCommitSha: commitSha,
buildWorkspace: workspace.root,
buildLastUsedAt: completedAt,
buildCompletedAt: completedAt,
buildError: undefined,
};
if (shouldRun) {
const started = await this.startProfile(candidate, assertLease);
const ready = started && (await this.waitForProfileReadiness(candidate, assertLease));
if (!ready) {
if (started) {
await this.stopProfile(candidate, assertLease);
}
const rollbackStarted =
(await this.startProfile(profile, assertLease)) &&
(await this.waitForProfileReadiness(profile, assertLease));
oldRuntimeStopped = !rollbackStarted;
const detail = rollbackStarted
? 'new profile release failed readiness; previous runtime restored'
: 'new profile release failed and previous runtime could not be restored';
await updateClaimedProfile({
buildStatus: 'FAILED',
buildCompletedAt: completedAt,
buildError: detail,
lastError: detail,
status: rollbackStarted ? profile.status : 'STOPPED',
});
return { ok: false, detail };
}
}
await assertLease();
await updateClaimedProfile({
buildStatus: 'SUCCEEDED',
buildCommitSha: commitSha,
buildWorkspace: workspace.root,
buildLastUsedAt: completedAt,
buildCompletedAt: completedAt,
buildError: null,
lastError: null,
});
oldRuntimeStopped = false;
return { ok: true };
} catch (error) {
if (error instanceof OperationLeaseLostError) {
throw error;
}
const detail = error instanceof Error ? error.message : String(error);
if (oldRuntimeStopped && shouldRun) {
try {
await this.startProfile(profile, assertLease);
} catch {
// The original error remains authoritative; reconciliation records the stopped runtime.
}
}
await updateClaimedProfile({
buildStatus: 'FAILED',
buildCompletedAt: this.now().toISOString(),
buildError: detail,
lastError: detail,
});
return { ok: false, detail };
} finally {
this.buildInFlight = false;
}
}
private async runAdminActionsNow(): Promise<void> {
if (this.stopping || this.adminActionInFlight) {
return;
@@ -1095,7 +1310,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
commitSha,
})
);
const { result, workspace } = await this.runBuildCommands(commitSha);
const { result, workspace } = await this.runBuildCommands(commitSha, profile);
await assertLease?.();
if (!result.ok) {
const completedAt = this.now().toISOString();
@@ -1289,12 +1504,18 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
return { databaseUrl, scenarioId, tickSeconds, meta };
}
private async runBuildCommands(commitSha: string): Promise<{
private async runBuildCommands(
commitSha: string,
profile?: GatewayProfileRecord
): Promise<{
result: Awaited<ReturnType<BuildRunner['run']>>;
workspace: Awaited<ReturnType<GitWorkspaceManager['prepare']>>;
}> {
const workspace = await this.workspaceManager.prepare(commitSha);
const commands = buildWorkspaceCommands(workspace.root, workspace.needsInstall, this.processConfig.baseEnv);
const commands = [
...buildWorkspaceCommands(workspace.root, workspace.needsInstall, this.processConfig.baseEnv),
...(profile ? buildProfileFrontendCommands(workspace.root, profile, this.processConfig.baseEnv) : []),
];
return { result: await this.buildRunner.run(commands), workspace };
}
@@ -1403,6 +1624,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
for (const [workspace, entry] of workspaceMap.entries()) {
const profileProcessNames = new Set(
entry.profileNames.flatMap((profileName) => [
buildProcessName(profileName, 'frontend'),
buildProcessName(profileName, 'api'),
buildProcessName(profileName, 'daemon'),
buildProcessName(profileName, 'auction'),
@@ -1451,6 +1673,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
private async startProfile(profile: GatewayProfileRecord, assertLease?: () => Promise<void>): Promise<boolean> {
const definitions = buildProcessDefinitions(profile, this.processConfig);
const orderedDefinitions = [
definitions.frontend,
definitions.api,
definitions.daemon,
definitions.auction,
@@ -1493,7 +1716,41 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
}
}
private async waitForProfileReadiness(
profile: GatewayProfileRecord,
assertLease?: () => Promise<void>
): Promise<boolean> {
const deadline = Date.now() + this.profileReadinessTimeoutMs;
const definitions = buildProcessDefinitions(profile, this.processConfig);
const expectedNames = Object.values(definitions).map((definition) => definition.name);
const apiUrl = `http://127.0.0.1:${profile.apiPort}/healthz`;
const frontendUrl = `http://127.0.0.1:${profile.apiPort - 1}/${profile.profile}/`;
while (Date.now() < deadline) {
await assertLease?.();
try {
const [api, frontend, processes] = await Promise.all([
this.fetchImpl(apiUrl),
this.fetchImpl(frontendUrl),
this.processManager.list(),
]);
const online = new Set(
processes
.filter((process) => process.status.toLowerCase() === 'online')
.map((process) => process.name)
);
if (api.ok && frontend.ok && expectedNames.every((name) => online.has(name))) {
return true;
}
} catch {
// Retry until the bounded deadline.
}
await new Promise<void>((resolve) => setTimeout(resolve, 500));
}
return false;
}
private async stopProfile(profile: GatewayProfileRecord, assertLease?: () => Promise<void>): Promise<void> {
const frontendName = buildProcessName(profile.profileName, 'frontend');
const apiName = buildProcessName(profile.profileName, 'api');
const daemonName = buildProcessName(profile.profileName, 'daemon');
const auctionName = buildProcessName(profile.profileName, 'auction');
@@ -1503,7 +1760,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
const existingNames = new Set((await this.processManager.list()).map((process) => process.name));
await assertLease?.();
const failures: string[] = [];
for (const name of [apiName, daemonName, auctionName, battleSimName, tournamentName]) {
for (const name of [frontendName, apiName, daemonName, auctionName, battleSimName, tournamentName]) {
if (!existingNames.has(name)) {
continue;
}
@@ -0,0 +1,323 @@
import type { GatewayPrisma, GatewayPrismaClient } from '@sammo-ts/infra';
import type { GatewayOperationStatus, GatewaySourceMode } from './profileRepository.js';
export const GATEWAY_RELEASE_OPERATION_TYPES = ['DEPLOY', 'ROLLBACK'] as const;
export type GatewayReleaseOperationType = (typeof GATEWAY_RELEASE_OPERATION_TYPES)[number];
export interface GatewayReleaseStateRecord {
id: string;
activeCommitSha?: string;
activeWorkspace?: string;
previousCommitSha?: string;
previousWorkspace?: string;
lastSuccessfulAt?: string;
lastError?: string;
updatedAt: string;
}
export interface GatewayReleaseOperationRecord {
id: string;
type: GatewayReleaseOperationType;
status: GatewayOperationStatus;
sourceMode?: GatewaySourceMode;
sourceRef?: string;
resolvedCommitSha?: string;
payload: GatewayPrisma.JsonObject;
reason?: string;
requestedBy: string;
startedAt?: string;
completedAt?: string;
error?: string;
leaseOwner?: string;
leaseUntil?: string;
heartbeatAt?: string;
attempts: number;
createdAt: string;
updatedAt: string;
}
export interface GatewayReleaseOperationCreateInput {
type: GatewayReleaseOperationType;
sourceMode?: GatewaySourceMode;
sourceRef?: string;
payload?: GatewayPrisma.JsonObject;
reason?: string;
requestedBy: string;
}
export interface GatewayReleaseRepository {
getState(): Promise<GatewayReleaseStateRecord>;
listOperations(limit?: number): Promise<GatewayReleaseOperationRecord[]>;
getOperation(id: string): Promise<GatewayReleaseOperationRecord | null>;
createOperation(input: GatewayReleaseOperationCreateInput): Promise<GatewayReleaseOperationRecord>;
claimNextOperation(
now: Date,
lease: { ownerId: string; durationMs: number }
): Promise<GatewayReleaseOperationRecord | null>;
renewOperationLease(id: string, ownerId: string, now: Date, durationMs: number): Promise<boolean>;
pinOperationResolvedCommit(id: string, ownerId: string, resolvedCommitSha: string): Promise<boolean>;
completeOperation(
id: string,
status: Extract<GatewayOperationStatus, 'SUCCEEDED' | 'FAILED'>,
fields: { resolvedCommitSha?: string | null; error?: string | null },
leaseOwner: string
): Promise<GatewayReleaseOperationRecord>;
publishRelease(
operationId: string,
leaseOwner: string,
release: { commitSha: string; workspace: string; previousCommitSha?: string; previousWorkspace?: string }
): Promise<GatewayReleaseStateRecord>;
recordStateError(detail: string): Promise<void>;
cancelOperation(id: string): Promise<boolean>;
retryOperation(id: string, requestedBy: string): Promise<GatewayReleaseOperationRecord | null>;
}
const toIso = (value: Date | null): string | undefined => (value ? value.toISOString() : undefined);
const mapState = (row: {
id: string;
activeCommitSha: string | null;
activeWorkspace: string | null;
previousCommitSha: string | null;
previousWorkspace: string | null;
lastSuccessfulAt: Date | null;
lastError: string | null;
updatedAt: Date;
}): GatewayReleaseStateRecord => ({
id: row.id,
activeCommitSha: row.activeCommitSha ?? undefined,
activeWorkspace: row.activeWorkspace ?? undefined,
previousCommitSha: row.previousCommitSha ?? undefined,
previousWorkspace: row.previousWorkspace ?? undefined,
lastSuccessfulAt: toIso(row.lastSuccessfulAt),
lastError: row.lastError ?? undefined,
updatedAt: row.updatedAt.toISOString(),
});
const mapOperation = (row: {
id: string;
type: GatewayReleaseOperationType;
status: GatewayOperationStatus;
sourceMode: GatewaySourceMode | null;
sourceRef: string | null;
resolvedCommitSha: string | null;
payload: GatewayPrisma.JsonValue;
reason: string | null;
requestedBy: string;
startedAt: Date | null;
completedAt: Date | null;
error: string | null;
leaseOwner: string | null;
leaseUntil: Date | null;
heartbeatAt: Date | null;
attempts: number;
createdAt: Date;
updatedAt: Date;
}): GatewayReleaseOperationRecord => ({
id: row.id,
type: row.type,
status: row.status,
sourceMode: row.sourceMode ?? undefined,
sourceRef: row.sourceRef ?? undefined,
resolvedCommitSha: row.resolvedCommitSha ?? undefined,
payload: (row.payload ?? {}) as GatewayPrisma.JsonObject,
reason: row.reason ?? undefined,
requestedBy: row.requestedBy,
startedAt: toIso(row.startedAt),
completedAt: toIso(row.completedAt),
error: row.error ?? undefined,
leaseOwner: row.leaseOwner ?? undefined,
leaseUntil: toIso(row.leaseUntil),
heartbeatAt: toIso(row.heartbeatAt),
attempts: row.attempts,
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
});
export const createGatewayReleaseRepository = (prisma: GatewayPrismaClient): GatewayReleaseRepository => ({
async getState() {
const row = await prisma.gatewayReleaseState.upsert({
where: { id: 'gateway' },
create: { id: 'gateway' },
update: {},
});
return mapState(row);
},
async listOperations(limit = 50) {
const rows = await prisma.gatewayReleaseOperation.findMany({
orderBy: { createdAt: 'desc' },
take: Math.min(Math.max(limit, 1), 200),
});
return rows.map(mapOperation);
},
async getOperation(id) {
const row = await prisma.gatewayReleaseOperation.findUnique({ where: { id } });
return row ? mapOperation(row) : null;
},
async createOperation(input) {
const row = await prisma.gatewayReleaseOperation.create({
data: {
type: input.type,
sourceMode: input.sourceMode,
sourceRef: input.sourceRef,
payload: input.payload ?? {},
reason: input.reason,
requestedBy: input.requestedBy,
},
});
return mapOperation(row);
},
async claimNextOperation(now, lease) {
const row = await prisma.$transaction(async (tx) => {
await tx.$queryRaw<Array<{ lock_result: string }>>`
SELECT pg_advisory_xact_lock(hashtextextended('gateway_release_operation_claim', 0))::text AS lock_result
`;
const staleBefore = new Date(now.getTime() - lease.durationMs);
const running = await tx.gatewayReleaseOperation.findFirst({
where: { status: 'RUNNING' },
orderBy: { createdAt: 'asc' },
});
const runningIsStale = Boolean(
running &&
((running.leaseUntil && running.leaseUntil < now) ||
(!running.leaseUntil && running.startedAt && running.startedAt <= staleBefore))
);
if (running && !runningIsStale) {
return null;
}
const candidate =
running ??
(await tx.gatewayReleaseOperation.findFirst({
where: { status: 'QUEUED' },
orderBy: { createdAt: 'asc' },
}));
if (!candidate) {
return null;
}
const claimed = await tx.gatewayReleaseOperation.updateMany({
where:
candidate.status === 'QUEUED'
? { id: candidate.id, status: 'QUEUED' }
: { id: candidate.id, status: 'RUNNING', leaseUntil: candidate.leaseUntil },
data: {
status: 'RUNNING',
startedAt: candidate.startedAt ?? now,
completedAt: null,
error: null,
leaseOwner: lease.ownerId,
leaseUntil: new Date(now.getTime() + lease.durationMs),
heartbeatAt: now,
attempts: { increment: 1 },
},
});
return claimed.count === 1 ? tx.gatewayReleaseOperation.findUnique({ where: { id: candidate.id } }) : null;
});
return row ? mapOperation(row) : null;
},
async renewOperationLease(id, ownerId, now, durationMs) {
const updated = await prisma.gatewayReleaseOperation.updateMany({
where: { id, status: 'RUNNING', leaseOwner: ownerId },
data: {
leaseUntil: new Date(now.getTime() + durationMs),
heartbeatAt: now,
},
});
return updated.count === 1;
},
async pinOperationResolvedCommit(id, ownerId, resolvedCommitSha) {
const updated = await prisma.gatewayReleaseOperation.updateMany({
where: {
id,
status: 'RUNNING',
leaseOwner: ownerId,
OR: [{ resolvedCommitSha: null }, { resolvedCommitSha }],
},
data: { resolvedCommitSha },
});
return updated.count === 1;
},
async completeOperation(id, status, fields, leaseOwner) {
const updated = await prisma.gatewayReleaseOperation.updateMany({
where: { id, status: 'RUNNING', leaseOwner },
data: {
status,
completedAt: new Date(),
resolvedCommitSha: fields.resolvedCommitSha,
error: fields.error,
leaseOwner: null,
leaseUntil: null,
heartbeatAt: null,
},
});
if (updated.count !== 1) {
throw new Error(`Gateway release lease lost before completion: ${id}`);
}
return mapOperation(await prisma.gatewayReleaseOperation.findUniqueOrThrow({ where: { id } }));
},
async publishRelease(operationId, leaseOwner, release) {
const row = await prisma.$transaction(async (tx) => {
const owned = await tx.gatewayReleaseOperation.findFirst({
where: { id: operationId, status: 'RUNNING', leaseOwner },
select: { id: true },
});
if (!owned) {
throw new Error(`Gateway release lease lost before publish: ${operationId}`);
}
return tx.gatewayReleaseState.upsert({
where: { id: 'gateway' },
create: {
id: 'gateway',
activeCommitSha: release.commitSha,
activeWorkspace: release.workspace,
previousCommitSha: release.previousCommitSha,
previousWorkspace: release.previousWorkspace,
lastSuccessfulAt: new Date(),
lastError: null,
},
update: {
activeCommitSha: release.commitSha,
activeWorkspace: release.workspace,
previousCommitSha: release.previousCommitSha,
previousWorkspace: release.previousWorkspace,
lastSuccessfulAt: new Date(),
lastError: null,
},
});
});
return mapState(row);
},
async recordStateError(detail) {
await prisma.gatewayReleaseState.upsert({
where: { id: 'gateway' },
create: { id: 'gateway', lastError: detail },
update: { lastError: detail },
});
},
async cancelOperation(id) {
const updated = await prisma.gatewayReleaseOperation.updateMany({
where: { id, status: 'QUEUED' },
data: { status: 'CANCELLED', completedAt: new Date() },
});
return updated.count === 1;
},
async retryOperation(id, requestedBy) {
const row = await prisma.$transaction(async (tx) => {
const previous = await tx.gatewayReleaseOperation.findUnique({ where: { id } });
if (!previous || (previous.status !== 'FAILED' && previous.status !== 'CANCELLED')) {
return null;
}
return tx.gatewayReleaseOperation.create({
data: {
type: previous.type,
sourceMode: previous.resolvedCommitSha ? 'COMMIT' : previous.sourceMode,
sourceRef: previous.resolvedCommitSha ?? previous.sourceRef,
payload: previous.payload as GatewayPrisma.JsonObject,
reason: previous.reason,
requestedBy,
},
});
});
return row ? mapOperation(row) : null;
},
});
@@ -14,7 +14,7 @@ export type GatewayProfileStatus = (typeof GATEWAY_PROFILE_STATUSES)[number];
export const GATEWAY_BUILD_STATUSES = ['IDLE', 'QUEUED', 'RUNNING', 'FAILED', 'SUCCEEDED'] as const;
export type GatewayBuildStatus = (typeof GATEWAY_BUILD_STATUSES)[number];
export const GATEWAY_OPERATION_TYPES = ['RESET', 'START', 'STOP'] as const;
export const GATEWAY_OPERATION_TYPES = ['RESET', 'DEPLOY', 'START', 'STOP'] as const;
export type GatewayOperationType = (typeof GATEWAY_OPERATION_TYPES)[number];
export const GATEWAY_OPERATION_STATUSES = ['QUEUED', 'RUNNING', 'SUCCEEDED', 'FAILED', 'CANCELLED'] as const;
@@ -0,0 +1,65 @@
import fs from 'node:fs/promises';
import path from 'node:path';
import { isRecord } from '@sammo-ts/common';
export const RELEASE_CONTROLLER_PROTOCOL = 1;
export interface ReleaseManifest {
formatVersion: 1;
controllerProtocol: number;
gatewaySchemaHead: string;
gameSchemaHead: string;
components: string[];
}
const assertMigrationHead = async (workspaceRoot: string, directory: string, expected: string): Promise<void> => {
const migrationRoot = path.join(workspaceRoot, 'packages', 'infra', 'prisma', directory);
const entries = await fs.readdir(migrationRoot, { withFileTypes: true });
const latest = entries
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort()
.at(-1);
if (!latest || latest !== expected) {
throw new Error(
`Release manifest ${directory} head ${expected} does not match workspace head ${latest ?? 'none'}.`
);
}
};
export const readReleaseManifest = async (workspaceRoot: string): Promise<ReleaseManifest> => {
const manifestPath = path.join(workspaceRoot, 'release-manifest.json');
const parsed = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as unknown;
if (
!isRecord(parsed) ||
parsed.formatVersion !== 1 ||
typeof parsed.controllerProtocol !== 'number' ||
!Number.isInteger(parsed.controllerProtocol) ||
typeof parsed.gatewaySchemaHead !== 'string' ||
typeof parsed.gameSchemaHead !== 'string' ||
!Array.isArray(parsed.components) ||
!parsed.components.every((component) => typeof component === 'string')
) {
throw new Error(`Invalid release manifest: ${manifestPath}`);
}
if (parsed.controllerProtocol > RELEASE_CONTROLLER_PROTOCOL) {
throw new Error(
`Release requires controller protocol ${parsed.controllerProtocol}; this controller supports ${RELEASE_CONTROLLER_PROTOCOL}.`
);
}
const manifest = parsed as unknown as ReleaseManifest;
await Promise.all([
assertMigrationHead(workspaceRoot, 'gateway-migrations', manifest.gatewaySchemaHead),
assertMigrationHead(workspaceRoot, 'migrations', manifest.gameSchemaHead),
]);
return manifest;
};
export const assertReleaseComponents = (manifest: ReleaseManifest, required: string[]): void => {
const available = new Set(manifest.components);
const missing = required.filter((component) => !available.has(component));
if (missing.length) {
throw new Error(`Release manifest is missing components: ${missing.join(', ')}`);
}
};
+3
View File
@@ -22,6 +22,7 @@ import { createPasswordHasher } from './auth/passwordHasher.js';
import { createPasswordEnvelopeService } from './auth/passwordEnvelope.js';
import { RedisGatewaySessionService } from './auth/redisSessionService.js';
import { createGatewayOrchestrator } from './orchestrator/orchestratorFactory.js';
import { createGatewayReleaseRepository } from './orchestrator/gatewayReleaseRepository.js';
import { appRouter } from './router.js';
import { RepositoryProfileStatusService } from './lobby/profileStatusService.js';
import { registerAccountIconInternalRoute } from './auth/accountIconInternalRoute.js';
@@ -64,6 +65,7 @@ export const createGatewayApiServer = async () => {
config,
process.env
);
const releases = createGatewayReleaseRepository(postgres.prisma as GatewayPrismaClient);
const profileStatus = new RepositoryProfileStatusService(profiles, orchestrator);
const app = fastify({
@@ -106,6 +108,7 @@ export const createGatewayApiServer = async () => {
localAccountGraceDays: config.localAccountGraceDays,
passwordEnvelope,
profiles,
releases,
orchestrator,
profileStatus,
requestHeaders: req.headers,