feat: Enhance Gateway Orchestrator with preopen and open scheduling, add error handling for paused states, and implement new profile management features
This commit is contained in:
@@ -54,7 +54,12 @@ export const planProfileReconcile = (
|
||||
status: GatewayProfileStatus,
|
||||
runtime: ProfileRuntimeState
|
||||
): { shouldStart: boolean; shouldStop: boolean } => {
|
||||
if (status === 'RUNNING') {
|
||||
if (
|
||||
status === 'RUNNING' ||
|
||||
status === 'PREOPEN' ||
|
||||
status === 'PAUSED' ||
|
||||
status === 'COMPLETED'
|
||||
) {
|
||||
return {
|
||||
shouldStart: !(runtime.apiRunning && runtime.daemonRunning),
|
||||
shouldStop: false,
|
||||
@@ -94,6 +99,7 @@ const buildProcessDefinitions = (
|
||||
TURN_PROFILE: profile.profile,
|
||||
PROFILE: profile.profile,
|
||||
SCENARIO: profile.scenario,
|
||||
TURN_PROFILE_NAME: profile.profileName,
|
||||
};
|
||||
return {
|
||||
api: {
|
||||
@@ -221,18 +227,41 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
const now = this.now();
|
||||
const due = await this.repository.listReservedToStart(now);
|
||||
for (const profile of due) {
|
||||
try {
|
||||
await this.repository.updateStatus(
|
||||
profile.profileName,
|
||||
'RUNNING',
|
||||
null
|
||||
);
|
||||
await this.startProfile(profile);
|
||||
} catch (error) {
|
||||
if (!profile.preopenAt || !profile.openAt) {
|
||||
await this.repository.updateLastError(
|
||||
profile.profileName,
|
||||
error instanceof Error ? error.message : 'Failed to start scheduled profile.'
|
||||
'Reserved profile is missing preopen/open schedule.'
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!profile.buildCommitSha) {
|
||||
await this.repository.updateLastError(
|
||||
profile.profileName,
|
||||
'Reserved profile is missing build commit SHA.'
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const queued =
|
||||
profile.buildStatus === 'QUEUED' || profile.buildStatus === 'RUNNING';
|
||||
if (!queued) {
|
||||
await this.repository.updateBuildStatus(profile.profileName, 'QUEUED', {
|
||||
requestedAt: now.toISOString(),
|
||||
error: null,
|
||||
commitSha: profile.buildCommitSha,
|
||||
});
|
||||
}
|
||||
}
|
||||
const profiles = await this.repository.listProfiles();
|
||||
for (const profile of profiles) {
|
||||
if (
|
||||
profile.status === 'PREOPEN' &&
|
||||
profile.openAt &&
|
||||
new Date(profile.openAt) <= now
|
||||
) {
|
||||
await this.repository.updateStatus(profile.profileName, 'RUNNING', {
|
||||
preopenAt: profile.preopenAt ?? null,
|
||||
openAt: profile.openAt ?? null,
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
@@ -304,12 +333,24 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
completedAt,
|
||||
error: null,
|
||||
});
|
||||
if (queued.status !== 'RUNNING' && queued.status !== 'DISABLED') {
|
||||
if (queued.status === 'RESERVED') {
|
||||
await this.repository.updateStatus(
|
||||
queued.profileName,
|
||||
'COMPLETED',
|
||||
queued.scheduledStartAt ? queued.scheduledStartAt : null
|
||||
queued.openAt && new Date(queued.openAt) <= this.now()
|
||||
? 'RUNNING'
|
||||
: 'PREOPEN',
|
||||
{
|
||||
preopenAt: queued.preopenAt ?? null,
|
||||
openAt: queued.openAt ?? null,
|
||||
}
|
||||
);
|
||||
} else if (queued.status === 'PREOPEN' && queued.openAt) {
|
||||
if (new Date(queued.openAt) <= this.now()) {
|
||||
await this.repository.updateStatus(queued.profileName, 'RUNNING', {
|
||||
preopenAt: queued.preopenAt ?? null,
|
||||
openAt: queued.openAt ?? null,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await this.repository.updateBuildStatus(queued.profileName, 'FAILED', {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import type { PrismaClient } from '@prisma/client';
|
||||
|
||||
import type { GatewayOrchestratorConfig } from '../config.js';
|
||||
import { createGatewayProfileRepository } from './profileRepository.js';
|
||||
import { GatewayOrchestrator } from './gatewayOrchestrator.js';
|
||||
import { Pm2ProcessManager } from './pm2ProcessManager.js';
|
||||
import { PnpmBuildRunner } from './buildRunner.js';
|
||||
import { resolveWorkspaceRoot } from './workspaceRoot.js';
|
||||
import { GitWorkspaceManager } from './workspaceManager.js';
|
||||
|
||||
export const buildEnvMap = (env: NodeJS.ProcessEnv): Record<string, string> => {
|
||||
const entries = Object.entries(env).filter(
|
||||
(entry): entry is [string, string] => typeof entry[1] === 'string'
|
||||
);
|
||||
return Object.fromEntries(entries);
|
||||
};
|
||||
|
||||
export const createGatewayOrchestrator = (
|
||||
prisma: PrismaClient,
|
||||
config: GatewayOrchestratorConfig,
|
||||
env: NodeJS.ProcessEnv = process.env
|
||||
): {
|
||||
orchestrator: GatewayOrchestrator;
|
||||
profiles: ReturnType<typeof createGatewayProfileRepository>;
|
||||
} => {
|
||||
const profiles = createGatewayProfileRepository(prisma);
|
||||
const workspaceRoot = resolveWorkspaceRoot(config.workspaceRootHint);
|
||||
const processManager = new Pm2ProcessManager();
|
||||
const buildRunner = new PnpmBuildRunner();
|
||||
const baseEnv = buildEnvMap(env);
|
||||
const workspaceManager = new GitWorkspaceManager({
|
||||
repoRoot: workspaceRoot,
|
||||
worktreeRoot: config.worktreeRoot,
|
||||
baseEnv,
|
||||
});
|
||||
const orchestrator = new GatewayOrchestrator({
|
||||
repository: profiles,
|
||||
processManager,
|
||||
buildRunner,
|
||||
workspaceManager,
|
||||
processConfig: {
|
||||
workspaceRoot,
|
||||
redisKeyPrefix: config.redisKeyPrefix,
|
||||
gameTokenSecret: config.gameTokenSecret,
|
||||
baseEnv,
|
||||
},
|
||||
reconcileIntervalMs: config.orchestratorReconcileIntervalMs,
|
||||
scheduleIntervalMs: config.orchestratorScheduleIntervalMs,
|
||||
buildIntervalMs: config.orchestratorBuildIntervalMs,
|
||||
});
|
||||
return { orchestrator, profiles };
|
||||
};
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { PrismaClient } from '@prisma/client';
|
||||
import {
|
||||
createPostgresConnector,
|
||||
resolvePostgresConfigFromEnv,
|
||||
} from '@sammo-ts/infra';
|
||||
|
||||
import { resolveGatewayOrchestratorConfigFromEnv } from '../config.js';
|
||||
import { createGatewayOrchestrator } from './orchestratorFactory.js';
|
||||
|
||||
export const runGatewayOrchestrator = async (): Promise<void> => {
|
||||
const config = resolveGatewayOrchestratorConfigFromEnv();
|
||||
const postgres = createPostgresConnector(resolvePostgresConfigFromEnv());
|
||||
await postgres.connect();
|
||||
|
||||
const { orchestrator } = createGatewayOrchestrator(
|
||||
postgres.prisma as PrismaClient,
|
||||
config,
|
||||
process.env
|
||||
);
|
||||
|
||||
const stop = async (reason: string): Promise<void> => {
|
||||
console.info(`[gateway-orchestrator] stopping: ${reason}`);
|
||||
await orchestrator.stop();
|
||||
await postgres.disconnect();
|
||||
};
|
||||
|
||||
process.on('SIGINT', () => void stop('SIGINT'));
|
||||
process.on('SIGTERM', () => void stop('SIGTERM'));
|
||||
|
||||
orchestrator.start();
|
||||
console.info('[gateway-orchestrator] started');
|
||||
};
|
||||
@@ -1,9 +1,11 @@
|
||||
import { Prisma, type PrismaClient } from '@prisma/client';
|
||||
|
||||
export const GATEWAY_PROFILE_STATUSES = [
|
||||
'COMPLETED',
|
||||
'RESERVED',
|
||||
'PREOPEN',
|
||||
'RUNNING',
|
||||
'PAUSED',
|
||||
'COMPLETED',
|
||||
'STOPPED',
|
||||
'DISABLED',
|
||||
] as const;
|
||||
@@ -28,6 +30,8 @@ export interface GatewayProfileRecord {
|
||||
buildCommitSha?: string;
|
||||
buildWorkspace?: string;
|
||||
buildLastUsedAt?: string;
|
||||
preopenAt?: string;
|
||||
openAt?: string;
|
||||
scheduledStartAt?: string;
|
||||
buildRequestedAt?: string;
|
||||
buildStartedAt?: string;
|
||||
@@ -44,7 +48,10 @@ export interface GatewayProfileUpsertInput {
|
||||
scenario: string;
|
||||
apiPort: number;
|
||||
status?: GatewayProfileStatus;
|
||||
preopenAt?: string;
|
||||
openAt?: string;
|
||||
scheduledStartAt?: string;
|
||||
buildCommitSha?: string;
|
||||
meta?: Prisma.JsonObject;
|
||||
}
|
||||
|
||||
@@ -55,7 +62,11 @@ export interface GatewayProfileRepository {
|
||||
updateStatus(
|
||||
profileName: string,
|
||||
status: GatewayProfileStatus,
|
||||
scheduledStartAt?: string | null
|
||||
schedule?: {
|
||||
preopenAt?: string | null;
|
||||
openAt?: string | null;
|
||||
scheduledStartAt?: string | null;
|
||||
}
|
||||
): Promise<GatewayProfileRecord | null>;
|
||||
updateBuildStatus(
|
||||
profileName: string,
|
||||
@@ -94,6 +105,8 @@ const mapProfile = (row: {
|
||||
buildCommitSha: string | null;
|
||||
buildWorkspace: string | null;
|
||||
buildLastUsedAt: Date | null;
|
||||
preopenAt: Date | null;
|
||||
openAt: Date | null;
|
||||
scheduledStartAt: Date | null;
|
||||
buildRequestedAt: Date | null;
|
||||
buildStartedAt: Date | null;
|
||||
@@ -113,6 +126,8 @@ const mapProfile = (row: {
|
||||
buildCommitSha: row.buildCommitSha ?? undefined,
|
||||
buildWorkspace: row.buildWorkspace ?? undefined,
|
||||
buildLastUsedAt: toIso(row.buildLastUsedAt),
|
||||
preopenAt: toIso(row.preopenAt),
|
||||
openAt: toIso(row.openAt),
|
||||
scheduledStartAt: toIso(row.scheduledStartAt),
|
||||
buildRequestedAt: toIso(row.buildRequestedAt),
|
||||
buildStartedAt: toIso(row.buildStartedAt),
|
||||
@@ -152,19 +167,36 @@ export const createGatewayProfileRepository = (
|
||||
scenario: input.scenario,
|
||||
apiPort: input.apiPort,
|
||||
status: input.status ?? 'STOPPED',
|
||||
preopenAt: input.preopenAt ? new Date(input.preopenAt) : null,
|
||||
openAt: input.openAt ? new Date(input.openAt) : null,
|
||||
scheduledStartAt: input.scheduledStartAt
|
||||
? new Date(input.scheduledStartAt)
|
||||
: null,
|
||||
buildCommitSha: input.buildCommitSha ?? null,
|
||||
meta: (input.meta ?? {}) as Prisma.JsonObject,
|
||||
},
|
||||
update: {
|
||||
apiPort: input.apiPort,
|
||||
status: input.status,
|
||||
preopenAt: input.preopenAt
|
||||
? new Date(input.preopenAt)
|
||||
: input.preopenAt === null
|
||||
? null
|
||||
: undefined,
|
||||
openAt: input.openAt
|
||||
? new Date(input.openAt)
|
||||
: input.openAt === null
|
||||
? null
|
||||
: undefined,
|
||||
scheduledStartAt: input.scheduledStartAt
|
||||
? new Date(input.scheduledStartAt)
|
||||
: input.scheduledStartAt === null
|
||||
? null
|
||||
: undefined,
|
||||
buildCommitSha:
|
||||
input.buildCommitSha === undefined
|
||||
? undefined
|
||||
: input.buildCommitSha,
|
||||
meta: input.meta ? (input.meta as Prisma.JsonObject) : undefined,
|
||||
},
|
||||
});
|
||||
@@ -173,17 +205,33 @@ export const createGatewayProfileRepository = (
|
||||
async updateStatus(
|
||||
profileName: string,
|
||||
status: GatewayProfileStatus,
|
||||
scheduledStartAt?: string | null
|
||||
schedule?: {
|
||||
preopenAt?: string | null;
|
||||
openAt?: string | null;
|
||||
scheduledStartAt?: string | null;
|
||||
}
|
||||
): Promise<GatewayProfileRecord | null> {
|
||||
const row = await prisma.gatewayProfile.update({
|
||||
where: { profileName },
|
||||
data: {
|
||||
status,
|
||||
scheduledStartAt:
|
||||
scheduledStartAt === undefined
|
||||
preopenAt:
|
||||
schedule?.preopenAt === undefined
|
||||
? undefined
|
||||
: scheduledStartAt
|
||||
? new Date(scheduledStartAt)
|
||||
: schedule?.preopenAt
|
||||
? new Date(schedule.preopenAt)
|
||||
: null,
|
||||
openAt:
|
||||
schedule?.openAt === undefined
|
||||
? undefined
|
||||
: schedule?.openAt
|
||||
? new Date(schedule.openAt)
|
||||
: null,
|
||||
scheduledStartAt:
|
||||
schedule?.scheduledStartAt === undefined
|
||||
? undefined
|
||||
: schedule?.scheduledStartAt
|
||||
? new Date(schedule.scheduledStartAt)
|
||||
: null,
|
||||
},
|
||||
});
|
||||
@@ -240,7 +288,7 @@ export const createGatewayProfileRepository = (
|
||||
const rows = await prisma.gatewayProfile.findMany({
|
||||
where: {
|
||||
status: 'RESERVED',
|
||||
scheduledStartAt: {
|
||||
preopenAt: {
|
||||
lte: now,
|
||||
},
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user