feat: Enhance Gateway Orchestrator with preopen and open scheduling, add error handling for paused states, and implement new profile management features

This commit is contained in:
2026-01-01 11:27:10 +00:00
parent e64962c3fd
commit 8c9f61700e
20 changed files with 470 additions and 86 deletions
@@ -54,7 +54,12 @@ export const planProfileReconcile = (
status: GatewayProfileStatus,
runtime: ProfileRuntimeState
): { shouldStart: boolean; shouldStop: boolean } => {
if (status === 'RUNNING') {
if (
status === 'RUNNING' ||
status === 'PREOPEN' ||
status === 'PAUSED' ||
status === 'COMPLETED'
) {
return {
shouldStart: !(runtime.apiRunning && runtime.daemonRunning),
shouldStop: false,
@@ -94,6 +99,7 @@ const buildProcessDefinitions = (
TURN_PROFILE: profile.profile,
PROFILE: profile.profile,
SCENARIO: profile.scenario,
TURN_PROFILE_NAME: profile.profileName,
};
return {
api: {
@@ -221,18 +227,41 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
const now = this.now();
const due = await this.repository.listReservedToStart(now);
for (const profile of due) {
try {
await this.repository.updateStatus(
profile.profileName,
'RUNNING',
null
);
await this.startProfile(profile);
} catch (error) {
if (!profile.preopenAt || !profile.openAt) {
await this.repository.updateLastError(
profile.profileName,
error instanceof Error ? error.message : 'Failed to start scheduled profile.'
'Reserved profile is missing preopen/open schedule.'
);
continue;
}
if (!profile.buildCommitSha) {
await this.repository.updateLastError(
profile.profileName,
'Reserved profile is missing build commit SHA.'
);
continue;
}
const queued =
profile.buildStatus === 'QUEUED' || profile.buildStatus === 'RUNNING';
if (!queued) {
await this.repository.updateBuildStatus(profile.profileName, 'QUEUED', {
requestedAt: now.toISOString(),
error: null,
commitSha: profile.buildCommitSha,
});
}
}
const profiles = await this.repository.listProfiles();
for (const profile of profiles) {
if (
profile.status === 'PREOPEN' &&
profile.openAt &&
new Date(profile.openAt) <= now
) {
await this.repository.updateStatus(profile.profileName, 'RUNNING', {
preopenAt: profile.preopenAt ?? null,
openAt: profile.openAt ?? null,
});
}
}
} finally {
@@ -304,12 +333,24 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
completedAt,
error: null,
});
if (queued.status !== 'RUNNING' && queued.status !== 'DISABLED') {
if (queued.status === 'RESERVED') {
await this.repository.updateStatus(
queued.profileName,
'COMPLETED',
queued.scheduledStartAt ? queued.scheduledStartAt : null
queued.openAt && new Date(queued.openAt) <= this.now()
? 'RUNNING'
: 'PREOPEN',
{
preopenAt: queued.preopenAt ?? null,
openAt: queued.openAt ?? null,
}
);
} else if (queued.status === 'PREOPEN' && queued.openAt) {
if (new Date(queued.openAt) <= this.now()) {
await this.repository.updateStatus(queued.profileName, 'RUNNING', {
preopenAt: queued.preopenAt ?? null,
openAt: queued.openAt ?? null,
});
}
}
} else {
await this.repository.updateBuildStatus(queued.profileName, 'FAILED', {
@@ -0,0 +1,52 @@
import type { PrismaClient } from '@prisma/client';
import type { GatewayOrchestratorConfig } from '../config.js';
import { createGatewayProfileRepository } from './profileRepository.js';
import { GatewayOrchestrator } from './gatewayOrchestrator.js';
import { Pm2ProcessManager } from './pm2ProcessManager.js';
import { PnpmBuildRunner } from './buildRunner.js';
import { resolveWorkspaceRoot } from './workspaceRoot.js';
import { GitWorkspaceManager } from './workspaceManager.js';
export const buildEnvMap = (env: NodeJS.ProcessEnv): Record<string, string> => {
const entries = Object.entries(env).filter(
(entry): entry is [string, string] => typeof entry[1] === 'string'
);
return Object.fromEntries(entries);
};
export const createGatewayOrchestrator = (
prisma: PrismaClient,
config: GatewayOrchestratorConfig,
env: NodeJS.ProcessEnv = process.env
): {
orchestrator: GatewayOrchestrator;
profiles: ReturnType<typeof createGatewayProfileRepository>;
} => {
const profiles = createGatewayProfileRepository(prisma);
const workspaceRoot = resolveWorkspaceRoot(config.workspaceRootHint);
const processManager = new Pm2ProcessManager();
const buildRunner = new PnpmBuildRunner();
const baseEnv = buildEnvMap(env);
const workspaceManager = new GitWorkspaceManager({
repoRoot: workspaceRoot,
worktreeRoot: config.worktreeRoot,
baseEnv,
});
const orchestrator = new GatewayOrchestrator({
repository: profiles,
processManager,
buildRunner,
workspaceManager,
processConfig: {
workspaceRoot,
redisKeyPrefix: config.redisKeyPrefix,
gameTokenSecret: config.gameTokenSecret,
baseEnv,
},
reconcileIntervalMs: config.orchestratorReconcileIntervalMs,
scheduleIntervalMs: config.orchestratorScheduleIntervalMs,
buildIntervalMs: config.orchestratorBuildIntervalMs,
});
return { orchestrator, profiles };
};
@@ -0,0 +1,32 @@
import type { PrismaClient } from '@prisma/client';
import {
createPostgresConnector,
resolvePostgresConfigFromEnv,
} from '@sammo-ts/infra';
import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js';
import { createGatewayOrchestrator } from './orchestratorFactory.js';
export const runGatewayOrchestrator = async (): Promise<void> => {
const config = resolveGatewayOrchestratorConfigFromEnv();
const postgres = createPostgresConnector(resolvePostgresConfigFromEnv());
await postgres.connect();
const { orchestrator } = createGatewayOrchestrator(
postgres.prisma as PrismaClient,
config,
process.env
);
const stop = async (reason: string): Promise<void> => {
console.info(`[gateway-orchestrator] stopping: ${reason}`);
await orchestrator.stop();
await postgres.disconnect();
};
process.on('SIGINT', () => void stop('SIGINT'));
process.on('SIGTERM', () => void stop('SIGTERM'));
orchestrator.start();
console.info('[gateway-orchestrator] started');
};
@@ -1,9 +1,11 @@
import { Prisma, type PrismaClient } from '@prisma/client';
export const GATEWAY_PROFILE_STATUSES = [
'COMPLETED',
'RESERVED',
'PREOPEN',
'RUNNING',
'PAUSED',
'COMPLETED',
'STOPPED',
'DISABLED',
] as const;
@@ -28,6 +30,8 @@ export interface GatewayProfileRecord {
buildCommitSha?: string;
buildWorkspace?: string;
buildLastUsedAt?: string;
preopenAt?: string;
openAt?: string;
scheduledStartAt?: string;
buildRequestedAt?: string;
buildStartedAt?: string;
@@ -44,7 +48,10 @@ export interface GatewayProfileUpsertInput {
scenario: string;
apiPort: number;
status?: GatewayProfileStatus;
preopenAt?: string;
openAt?: string;
scheduledStartAt?: string;
buildCommitSha?: string;
meta?: Prisma.JsonObject;
}
@@ -55,7 +62,11 @@ export interface GatewayProfileRepository {
updateStatus(
profileName: string,
status: GatewayProfileStatus,
scheduledStartAt?: string | null
schedule?: {
preopenAt?: string | null;
openAt?: string | null;
scheduledStartAt?: string | null;
}
): Promise<GatewayProfileRecord | null>;
updateBuildStatus(
profileName: string,
@@ -94,6 +105,8 @@ const mapProfile = (row: {
buildCommitSha: string | null;
buildWorkspace: string | null;
buildLastUsedAt: Date | null;
preopenAt: Date | null;
openAt: Date | null;
scheduledStartAt: Date | null;
buildRequestedAt: Date | null;
buildStartedAt: Date | null;
@@ -113,6 +126,8 @@ const mapProfile = (row: {
buildCommitSha: row.buildCommitSha ?? undefined,
buildWorkspace: row.buildWorkspace ?? undefined,
buildLastUsedAt: toIso(row.buildLastUsedAt),
preopenAt: toIso(row.preopenAt),
openAt: toIso(row.openAt),
scheduledStartAt: toIso(row.scheduledStartAt),
buildRequestedAt: toIso(row.buildRequestedAt),
buildStartedAt: toIso(row.buildStartedAt),
@@ -152,19 +167,36 @@ export const createGatewayProfileRepository = (
scenario: input.scenario,
apiPort: input.apiPort,
status: input.status ?? 'STOPPED',
preopenAt: input.preopenAt ? new Date(input.preopenAt) : null,
openAt: input.openAt ? new Date(input.openAt) : null,
scheduledStartAt: input.scheduledStartAt
? new Date(input.scheduledStartAt)
: null,
buildCommitSha: input.buildCommitSha ?? null,
meta: (input.meta ?? {}) as Prisma.JsonObject,
},
update: {
apiPort: input.apiPort,
status: input.status,
preopenAt: input.preopenAt
? new Date(input.preopenAt)
: input.preopenAt === null
? null
: undefined,
openAt: input.openAt
? new Date(input.openAt)
: input.openAt === null
? null
: undefined,
scheduledStartAt: input.scheduledStartAt
? new Date(input.scheduledStartAt)
: input.scheduledStartAt === null
? null
: undefined,
buildCommitSha:
input.buildCommitSha === undefined
? undefined
: input.buildCommitSha,
meta: input.meta ? (input.meta as Prisma.JsonObject) : undefined,
},
});
@@ -173,17 +205,33 @@ export const createGatewayProfileRepository = (
async updateStatus(
profileName: string,
status: GatewayProfileStatus,
scheduledStartAt?: string | null
schedule?: {
preopenAt?: string | null;
openAt?: string | null;
scheduledStartAt?: string | null;
}
): Promise<GatewayProfileRecord | null> {
const row = await prisma.gatewayProfile.update({
where: { profileName },
data: {
status,
scheduledStartAt:
scheduledStartAt === undefined
preopenAt:
schedule?.preopenAt === undefined
? undefined
: scheduledStartAt
? new Date(scheduledStartAt)
: schedule?.preopenAt
? new Date(schedule.preopenAt)
: null,
openAt:
schedule?.openAt === undefined
? undefined
: schedule?.openAt
? new Date(schedule.openAt)
: null,
scheduledStartAt:
schedule?.scheduledStartAt === undefined
? undefined
: schedule?.scheduledStartAt
? new Date(schedule.scheduledStartAt)
: null,
},
});
@@ -240,7 +288,7 @@ export const createGatewayProfileRepository = (
const rows = await prisma.gatewayProfile.findMany({
where: {
status: 'RESERVED',
scheduledStartAt: {
preopenAt: {
lte: now,
},
},