feat: 게이트웨이 시계 생명주기 권위 연결
This commit is contained in:
@@ -2750,6 +2750,19 @@ export const adminRouter = router({
|
|||||||
};
|
};
|
||||||
await ctx.profiles.updateMeta(input.profileName, nextMeta);
|
await ctx.profiles.updateMeta(input.profileName, nextMeta);
|
||||||
if (mappedStatus) {
|
if (mappedStatus) {
|
||||||
|
if (input.action === 'PAUSE' || input.action === 'STOP') {
|
||||||
|
await ctx.orchestrator.transitionProfileClock(
|
||||||
|
input.profileName,
|
||||||
|
'SUSPEND',
|
||||||
|
input.reason?.trim() || `gateway ${input.action.toLowerCase()} by ${adminAuth.user.id}`
|
||||||
|
);
|
||||||
|
} else if (input.action === 'RESUME') {
|
||||||
|
await ctx.orchestrator.transitionProfileClock(
|
||||||
|
input.profileName,
|
||||||
|
'RESUME',
|
||||||
|
input.reason?.trim() || `gateway resume by ${adminAuth.user.id}`
|
||||||
|
);
|
||||||
|
}
|
||||||
await ctx.profiles.updateStatus(input.profileName, mappedStatus);
|
await ctx.profiles.updateStatus(input.profileName, mappedStatus);
|
||||||
await ctx.orchestrator.reconcileNow();
|
await ctx.orchestrator.reconcileNow();
|
||||||
const appliedActionRecord = {
|
const appliedActionRecord = {
|
||||||
|
|||||||
@@ -5,6 +5,12 @@ import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
|||||||
import { stripVTControlCharacters } from 'node:util';
|
import { stripVTControlCharacters } from 'node:util';
|
||||||
|
|
||||||
import type { ScenarioInstallOptions } from '@sammo-ts/game-engine/scenario/scenarioSeeder.js';
|
import type { ScenarioInstallOptions } from '@sammo-ts/game-engine/scenario/scenarioSeeder.js';
|
||||||
|
import {
|
||||||
|
applyNextClockProjection,
|
||||||
|
reconcileClockSuspension,
|
||||||
|
startClockSuspension,
|
||||||
|
type ClockOperationAuthority,
|
||||||
|
} from '@sammo-ts/game-engine';
|
||||||
import {
|
import {
|
||||||
cancelGame as defaultCancelGame,
|
cancelGame as defaultCancelGame,
|
||||||
GAME_CANCELLATION_GENERAL_MODES,
|
GAME_CANCELLATION_GENERAL_MODES,
|
||||||
@@ -17,6 +23,9 @@ import { gatewayProfileCapabilities } from '@sammo-ts/common';
|
|||||||
import {
|
import {
|
||||||
createGamePostgresConnector,
|
createGamePostgresConnector,
|
||||||
createRedisConnector,
|
createRedisConnector,
|
||||||
|
CLOCK_OPERATION_PERSISTENCE_LOCK,
|
||||||
|
GamePrisma,
|
||||||
|
acquireGameSchemaAdvisoryXactLock,
|
||||||
resolvePostgresPoolMax,
|
resolvePostgresPoolMax,
|
||||||
resolveRedisConfigFromEnv,
|
resolveRedisConfigFromEnv,
|
||||||
} from '@sammo-ts/infra';
|
} from '@sammo-ts/infra';
|
||||||
@@ -112,6 +121,12 @@ export interface GatewayOrchestratorOptions {
|
|||||||
fetchImpl?: typeof fetch;
|
fetchImpl?: typeof fetch;
|
||||||
clearTournamentRuntimeState?: (profileName: string) => Promise<void>;
|
clearTournamentRuntimeState?: (profileName: string) => Promise<void>;
|
||||||
cancelGame?: typeof defaultCancelGame;
|
cancelGame?: typeof defaultCancelGame;
|
||||||
|
transitionProfileClock?: (
|
||||||
|
profileName: string,
|
||||||
|
action: 'SUSPEND' | 'RESUME',
|
||||||
|
reason: string
|
||||||
|
) => Promise<{ phase: string; revision: number }>;
|
||||||
|
promoteProfileOpening?: (profile: GatewayProfileRecord) => Promise<void>;
|
||||||
}
|
}
|
||||||
|
|
||||||
const WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
const WORKSPACE_CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
||||||
@@ -161,6 +176,11 @@ export interface GatewayOrchestratorHandle {
|
|||||||
}>;
|
}>;
|
||||||
listRuntimeStates(profileNames: string[]): Promise<ProfileRuntimeSnapshot[]>;
|
listRuntimeStates(profileNames: string[]): Promise<ProfileRuntimeSnapshot[]>;
|
||||||
listRuntimeSettings?(profileNames: string[]): Promise<ProfileRuntimeSettingsSnapshot[]>;
|
listRuntimeSettings?(profileNames: string[]): Promise<ProfileRuntimeSettingsSnapshot[]>;
|
||||||
|
transitionProfileClock(
|
||||||
|
profileName: string,
|
||||||
|
action: 'SUSPEND' | 'RESUME',
|
||||||
|
reason: string
|
||||||
|
): Promise<{ phase: string; revision: number }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface GatewayManagedCleanupResult {
|
export interface GatewayManagedCleanupResult {
|
||||||
@@ -322,6 +342,14 @@ const readMetaNumber = (meta: Record<string, unknown>, key: string): number | nu
|
|||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const clockRevisionAsNumber = (revision: bigint): number => {
|
||||||
|
const value = Number(revision);
|
||||||
|
if (!Number.isSafeInteger(value) || value < 1) {
|
||||||
|
throw new Error(`Clock revision is outside the safe API integer range: ${revision.toString()}.`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
};
|
||||||
|
|
||||||
export const resolveProfileFirstGameIdx = (meta: Record<string, unknown>): number => {
|
export const resolveProfileFirstGameIdx = (meta: Record<string, unknown>): number => {
|
||||||
const raw = meta.firstGameIdx;
|
const raw = meta.firstGameIdx;
|
||||||
const configured = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : Number.NaN;
|
const configured = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : Number.NaN;
|
||||||
@@ -842,10 +870,7 @@ const assertProfileMigrationEnvironmentTimeZone = (env?: Record<string, string>)
|
|||||||
if (pgTimeZone) assertProfileMigrationTimeZone(pgTimeZone, 'PGTZ');
|
if (pgTimeZone) assertProfileMigrationTimeZone(pgTimeZone, 'PGTZ');
|
||||||
};
|
};
|
||||||
|
|
||||||
const buildProfileMigrationEnv = (
|
const buildProfileMigrationEnv = (profileDatabaseUrl: string, env?: Record<string, string>): Record<string, string> => {
|
||||||
profileDatabaseUrl: string,
|
|
||||||
env?: Record<string, string>
|
|
||||||
): Record<string, string> => {
|
|
||||||
assertProfileMigrationEnvironmentTimeZone(env);
|
assertProfileMigrationEnvironmentTimeZone(env);
|
||||||
return { ...(env ?? {}), DATABASE_URL: buildProfileMigrationDatabaseUrl(profileDatabaseUrl) };
|
return { ...(env ?? {}), DATABASE_URL: buildProfileMigrationDatabaseUrl(profileDatabaseUrl) };
|
||||||
};
|
};
|
||||||
@@ -942,6 +967,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
private readonly fetchImpl: typeof fetch;
|
private readonly fetchImpl: typeof fetch;
|
||||||
private readonly clearTournamentRuntimeState: (profileName: string) => Promise<void>;
|
private readonly clearTournamentRuntimeState: (profileName: string) => Promise<void>;
|
||||||
private readonly cancelGame: typeof defaultCancelGame;
|
private readonly cancelGame: typeof defaultCancelGame;
|
||||||
|
private readonly transitionProfileClockOverride?: GatewayOrchestratorOptions['transitionProfileClock'];
|
||||||
|
private readonly promoteProfileOpeningOverride?: GatewayOrchestratorOptions['promoteProfileOpening'];
|
||||||
private reconcileTimer?: NodeJS.Timeout;
|
private reconcileTimer?: NodeJS.Timeout;
|
||||||
private scheduleTimer?: NodeJS.Timeout;
|
private scheduleTimer?: NodeJS.Timeout;
|
||||||
private buildTimer?: NodeJS.Timeout;
|
private buildTimer?: NodeJS.Timeout;
|
||||||
@@ -986,6 +1013,8 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
options.clearTournamentRuntimeState ??
|
options.clearTournamentRuntimeState ??
|
||||||
((profileName) => this.clearTournamentRuntimeStateFromRedis(profileName));
|
((profileName) => this.clearTournamentRuntimeStateFromRedis(profileName));
|
||||||
this.cancelGame = options.cancelGame ?? defaultCancelGame;
|
this.cancelGame = options.cancelGame ?? defaultCancelGame;
|
||||||
|
this.transitionProfileClockOverride = options.transitionProfileClock;
|
||||||
|
this.promoteProfileOpeningOverride = options.promoteProfileOpening;
|
||||||
}
|
}
|
||||||
|
|
||||||
private sanitizeOperationLogMessage(message: string): string {
|
private sanitizeOperationLogMessage(message: string): string {
|
||||||
@@ -1171,6 +1200,191 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
return snapshots.filter((snapshot): snapshot is ProfileRuntimeSettingsSnapshot => snapshot !== null);
|
return snapshots.filter((snapshot): snapshot is ProfileRuntimeSettingsSnapshot => snapshot !== null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async transitionProfileClock(
|
||||||
|
profileName: string,
|
||||||
|
action: 'SUSPEND' | 'RESUME',
|
||||||
|
reason: string
|
||||||
|
): Promise<{ phase: string; revision: number }> {
|
||||||
|
if (this.transitionProfileClockOverride) {
|
||||||
|
return this.transitionProfileClockOverride(profileName, action, reason);
|
||||||
|
}
|
||||||
|
const profile = await this.repository.getProfile(profileName);
|
||||||
|
if (!profile || profile.currentScenario === null) {
|
||||||
|
throw new Error(`Profile clock is unavailable: ${profileName}`);
|
||||||
|
}
|
||||||
|
const postgres = createGamePostgresConnector({ url: this.resolveProfileDatabaseUrl(profile) });
|
||||||
|
const redis = createRedisConnector(resolveRedisConfigFromEnv(this.processConfig.baseEnv ?? process.env));
|
||||||
|
await postgres.connect();
|
||||||
|
await redis.connect();
|
||||||
|
try {
|
||||||
|
const authorityRows = await postgres.prisma.$queryRaw<
|
||||||
|
Array<{ ownerId: string; fencingEpoch: bigint; valid: boolean }>
|
||||||
|
>(GamePrisma.sql`
|
||||||
|
SELECT owner_id AS "ownerId",
|
||||||
|
fencing_epoch AS "fencingEpoch",
|
||||||
|
lease_until > (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') AS valid
|
||||||
|
FROM turn_daemon_lease
|
||||||
|
WHERE profile = ${profileName}
|
||||||
|
`);
|
||||||
|
const liveLease = authorityRows.find((row) => row.valid);
|
||||||
|
const authority: ClockOperationAuthority = liveLease
|
||||||
|
? {
|
||||||
|
kind: 'DAEMON',
|
||||||
|
profileName,
|
||||||
|
ownerId: liveLease.ownerId,
|
||||||
|
fencingEpoch: liveLease.fencingEpoch,
|
||||||
|
}
|
||||||
|
: { kind: 'OFFLINE', profileName, reason };
|
||||||
|
const world = await postgres.prisma.worldState.findFirst({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { clockPhase: true, clockRevision: true, deadlineGeneration: true },
|
||||||
|
});
|
||||||
|
if (!world) throw new Error(`Profile has no world_state: ${profileName}`);
|
||||||
|
|
||||||
|
if (action === 'SUSPEND') {
|
||||||
|
let suspension = await postgres.prisma.clockSuspension.findFirst({
|
||||||
|
where: { sourceRevision: world.clockRevision, status: 'SUSPENDED' },
|
||||||
|
orderBy: { createdAt: 'desc' },
|
||||||
|
});
|
||||||
|
if (world.clockPhase === 'RUNNING') {
|
||||||
|
const suffix = createHash('sha256')
|
||||||
|
.update(`${profileName}:${world.clockRevision.toString()}`)
|
||||||
|
.digest('hex')
|
||||||
|
.slice(0, 20);
|
||||||
|
const started = await startClockSuspension({
|
||||||
|
db: postgres.prisma,
|
||||||
|
suspensionId: `gateway-maintenance-${suffix}`,
|
||||||
|
source: 'MAINTENANCE',
|
||||||
|
authority,
|
||||||
|
});
|
||||||
|
suspension = await postgres.prisma.clockSuspension.findUniqueOrThrow({
|
||||||
|
where: { id: started.suspensionId },
|
||||||
|
});
|
||||||
|
} else if (world.clockPhase !== 'SUSPENDED') {
|
||||||
|
throw new Error(`Cannot suspend profile clock from ${world.clockPhase}.`);
|
||||||
|
}
|
||||||
|
if (!suspension) throw new Error('Suspended profile is missing its durable clock ledger.');
|
||||||
|
const phaseResult = await redis.client.eval(
|
||||||
|
`
|
||||||
|
local active = redis.call('GET', KEYS[1])
|
||||||
|
if active and active ~= ARGV[1] then return 0 end
|
||||||
|
redis.call('SET', KEYS[1], ARGV[1])
|
||||||
|
redis.call('SET', KEYS[2], ARGV[2])
|
||||||
|
redis.call('SET', KEYS[3], 'SUSPENDED')
|
||||||
|
return 1
|
||||||
|
`,
|
||||||
|
{
|
||||||
|
keys: [
|
||||||
|
`sammo:${profileName}:clock:active-revision`,
|
||||||
|
`sammo:${profileName}:clock:deadline-generation`,
|
||||||
|
`sammo:${profileName}:clock:phase`,
|
||||||
|
],
|
||||||
|
arguments: [world.clockRevision.toString(), world.deadlineGeneration.toString()],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if (Number(phaseResult) !== 1) throw new Error('Redis clock source revision differs from the DB.');
|
||||||
|
return { phase: 'SUSPENDED', revision: clockRevisionAsNumber(suspension.sourceRevision) };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (world.clockPhase === 'RUNNING') {
|
||||||
|
return { phase: 'RUNNING', revision: clockRevisionAsNumber(world.clockRevision) };
|
||||||
|
}
|
||||||
|
const suspension = await postgres.prisma.clockSuspension.findFirst({
|
||||||
|
where: { status: { in: ['SUSPENDED', 'RECONCILING'] } },
|
||||||
|
orderBy: { createdAt: 'desc' },
|
||||||
|
});
|
||||||
|
if (!suspension) throw new Error('Profile resume requires a durable suspended clock ledger.');
|
||||||
|
if (world.clockPhase === 'SUSPENDED') {
|
||||||
|
await reconcileClockSuspension({ db: postgres.prisma, suspensionId: suspension.id, authority });
|
||||||
|
} else if (world.clockPhase !== 'RECONCILING') {
|
||||||
|
throw new Error(`Cannot resume profile clock from ${world.clockPhase}.`);
|
||||||
|
}
|
||||||
|
const projected = await applyNextClockProjection({
|
||||||
|
db: postgres.prisma,
|
||||||
|
redis: redis.client,
|
||||||
|
workerId: `gateway:${this.operationLeaseOwner}`,
|
||||||
|
});
|
||||||
|
if (projected === 'IDLE') throw new Error('Clock reconciliation has no claimable projection outbox.');
|
||||||
|
const resumed = await postgres.prisma.worldState.findFirstOrThrow({
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
select: { clockPhase: true, clockRevision: true },
|
||||||
|
});
|
||||||
|
if (resumed.clockPhase !== 'RUNNING') throw new Error('Clock projection did not reach RUNNING.');
|
||||||
|
return { phase: resumed.clockPhase, revision: clockRevisionAsNumber(resumed.clockRevision) };
|
||||||
|
} finally {
|
||||||
|
await redis.disconnect().catch(() => undefined);
|
||||||
|
await postgres.disconnect().catch(() => undefined);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async promoteProfileOpening(profile: GatewayProfileRecord): Promise<void> {
|
||||||
|
if (this.promoteProfileOpeningOverride) {
|
||||||
|
await this.promoteProfileOpeningOverride(profile);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const postgres = createGamePostgresConnector({ url: this.resolveProfileDatabaseUrl(profile) });
|
||||||
|
const redis = createRedisConnector(resolveRedisConfigFromEnv(this.processConfig.baseEnv ?? process.env));
|
||||||
|
await postgres.connect();
|
||||||
|
await redis.connect();
|
||||||
|
try {
|
||||||
|
const clock = await postgres.prisma.$transaction(async (transaction) => {
|
||||||
|
await acquireGameSchemaAdvisoryXactLock(transaction, CLOCK_OPERATION_PERSISTENCE_LOCK);
|
||||||
|
const rows = await transaction.$queryRaw<Array<{ id: number }>>(GamePrisma.sql`
|
||||||
|
SELECT id FROM world_state ORDER BY id LIMIT 2 FOR UPDATE
|
||||||
|
`);
|
||||||
|
if (rows.length !== 1) {
|
||||||
|
throw new Error(`Opening promotion requires exactly one world_state row; found ${rows.length}.`);
|
||||||
|
}
|
||||||
|
const world = await transaction.worldState.findUniqueOrThrow({ where: { id: rows[0]!.id } });
|
||||||
|
if (world.clockPhase === 'RUNNING') {
|
||||||
|
return { revision: world.clockRevision, generation: world.deadlineGeneration };
|
||||||
|
}
|
||||||
|
if (world.clockPhase !== 'PREOPEN' || world.clockTick !== 0n || !world.clockWallAnchor) {
|
||||||
|
throw new Error(
|
||||||
|
`Opening promotion requires PREOPEN at tick 0; found ${world.clockPhase}@${world.clockTick?.toString() ?? 'null'}.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const [wall] = await transaction.$queryRaw<Array<{ wallNow: Date }>>(GamePrisma.sql`
|
||||||
|
SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::timestamp(3) AS "wallNow"
|
||||||
|
`);
|
||||||
|
if (!wall || wall.wallNow.getTime() < world.clockWallAnchor.getTime()) {
|
||||||
|
throw new Error('Opening promotion was requested before the scheduled wall instant.');
|
||||||
|
}
|
||||||
|
await transaction.worldState.update({
|
||||||
|
where: { id: world.id },
|
||||||
|
data: { clockPhase: 'RUNNING' },
|
||||||
|
});
|
||||||
|
return { revision: world.clockRevision, generation: world.deadlineGeneration };
|
||||||
|
});
|
||||||
|
const result = await redis.client.eval(
|
||||||
|
`
|
||||||
|
local revision = redis.call('GET', KEYS[1])
|
||||||
|
local generation = redis.call('GET', KEYS[2])
|
||||||
|
if revision and revision ~= ARGV[1] then return 0 end
|
||||||
|
if generation and generation ~= ARGV[2] then return 0 end
|
||||||
|
redis.call('SET', KEYS[1], ARGV[1])
|
||||||
|
redis.call('SET', KEYS[2], ARGV[2])
|
||||||
|
redis.call('SET', KEYS[3], 'RUNNING')
|
||||||
|
return 1
|
||||||
|
`,
|
||||||
|
{
|
||||||
|
keys: [
|
||||||
|
`sammo:${profile.profileName}:clock:active-revision`,
|
||||||
|
`sammo:${profile.profileName}:clock:deadline-generation`,
|
||||||
|
`sammo:${profile.profileName}:clock:phase`,
|
||||||
|
],
|
||||||
|
arguments: [clock.revision.toString(), clock.generation.toString()],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if (Number(result) !== 1) {
|
||||||
|
throw new Error('Opening promotion found a different Redis clock revision or generation.');
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await redis.disconnect().catch(() => undefined);
|
||||||
|
await postgres.disconnect().catch(() => undefined);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async reconcileNow(): Promise<void> {
|
async reconcileNow(): Promise<void> {
|
||||||
if (this.stopping || this.reconcileInFlight) {
|
if (this.stopping || this.reconcileInFlight) {
|
||||||
return;
|
return;
|
||||||
@@ -1232,14 +1446,14 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (profile.currentScenario !== null && profile.buildStatus === 'SUCCEEDED' && profile.buildWorkspace) {
|
if (profile.currentScenario !== null && profile.buildStatus === 'SUCCEEDED' && profile.buildWorkspace) {
|
||||||
await this.repository.updateStatus(
|
const nextStatus = resolveResetLifecycleStatus(now, preopenAt, openAt);
|
||||||
profile.profileName,
|
if (nextStatus === 'RUNNING') {
|
||||||
resolveResetLifecycleStatus(now, preopenAt, openAt),
|
await this.promoteProfileOpening(profile);
|
||||||
{
|
}
|
||||||
preopenAt: profile.preopenAt,
|
await this.repository.updateStatus(profile.profileName, nextStatus, {
|
||||||
openAt: profile.openAt,
|
preopenAt: profile.preopenAt,
|
||||||
}
|
openAt: profile.openAt,
|
||||||
);
|
});
|
||||||
await this.repository.updateLastError(profile.profileName, null);
|
await this.repository.updateLastError(profile.profileName, null);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -1255,6 +1469,7 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
const profiles = await this.repository.listProfiles();
|
const profiles = await this.repository.listProfiles();
|
||||||
for (const profile of profiles) {
|
for (const profile of profiles) {
|
||||||
if (profile.status === 'PREOPEN' && profile.openAt && new Date(profile.openAt) <= now) {
|
if (profile.status === 'PREOPEN' && profile.openAt && new Date(profile.openAt) <= now) {
|
||||||
|
await this.promoteProfileOpening(profile);
|
||||||
await this.repository.updateStatus(profile.profileName, 'RUNNING', {
|
await this.repository.updateStatus(profile.profileName, 'RUNNING', {
|
||||||
preopenAt: profile.preopenAt ?? null,
|
preopenAt: profile.preopenAt ?? null,
|
||||||
openAt: profile.openAt ?? null,
|
openAt: profile.openAt ?? null,
|
||||||
@@ -1301,16 +1516,17 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
|||||||
error: null,
|
error: null,
|
||||||
});
|
});
|
||||||
if (queued.status === 'RESERVED') {
|
if (queued.status === 'RESERVED') {
|
||||||
await this.repository.updateStatus(
|
const opensImmediately = Boolean(queued.openAt && new Date(queued.openAt) <= this.now());
|
||||||
queued.profileName,
|
if (opensImmediately) {
|
||||||
queued.openAt && new Date(queued.openAt) <= this.now() ? 'RUNNING' : 'PREOPEN',
|
await this.promoteProfileOpening(queued);
|
||||||
{
|
}
|
||||||
preopenAt: queued.preopenAt ?? null,
|
await this.repository.updateStatus(queued.profileName, opensImmediately ? 'RUNNING' : 'PREOPEN', {
|
||||||
openAt: queued.openAt ?? null,
|
preopenAt: queued.preopenAt ?? null,
|
||||||
}
|
openAt: queued.openAt ?? null,
|
||||||
);
|
});
|
||||||
} else if (queued.status === 'PREOPEN' && queued.openAt) {
|
} else if (queued.status === 'PREOPEN' && queued.openAt) {
|
||||||
if (new Date(queued.openAt) <= this.now()) {
|
if (new Date(queued.openAt) <= this.now()) {
|
||||||
|
await this.promoteProfileOpening(queued);
|
||||||
await this.repository.updateStatus(queued.profileName, 'RUNNING', {
|
await this.repository.updateStatus(queued.profileName, 'RUNNING', {
|
||||||
preopenAt: queued.preopenAt ?? null,
|
preopenAt: queued.preopenAt ?? null,
|
||||||
openAt: queued.openAt ?? null,
|
openAt: queued.openAt ?? null,
|
||||||
|
|||||||
@@ -87,6 +87,7 @@ const buildCaller = async (
|
|||||||
const updatedStatuses: GatewayProfileRecord['status'][] = [];
|
const updatedStatuses: GatewayProfileRecord['status'][] = [];
|
||||||
const updatedMetas: Record<string, unknown>[] = [];
|
const updatedMetas: Record<string, unknown>[] = [];
|
||||||
const auditEvents: AdminAuditEventRecord[] = [];
|
const auditEvents: AdminAuditEventRecord[] = [];
|
||||||
|
const lifecycle: string[] = [];
|
||||||
let reconcileCount = 0;
|
let reconcileCount = 0;
|
||||||
let runtimeStateListCount = 0;
|
let runtimeStateListCount = 0;
|
||||||
let storedNotice = options.initialNotice ?? '';
|
let storedNotice = options.initialNotice ?? '';
|
||||||
@@ -111,6 +112,7 @@ const buildCaller = async (
|
|||||||
updateCurrentScenario: async () => profile,
|
updateCurrentScenario: async () => profile,
|
||||||
updateStatus: async (_profileName, status) => {
|
updateStatus: async (_profileName, status) => {
|
||||||
updatedStatuses.push(status);
|
updatedStatuses.push(status);
|
||||||
|
lifecycle.push(`status:${status}`);
|
||||||
return { ...profile, status };
|
return { ...profile, status };
|
||||||
},
|
},
|
||||||
updateBuildStatus: async () => profile,
|
updateBuildStatus: async () => profile,
|
||||||
@@ -288,6 +290,7 @@ const buildCaller = async (
|
|||||||
stop: async () => {},
|
stop: async () => {},
|
||||||
reconcileNow: async () => {
|
reconcileNow: async () => {
|
||||||
reconcileCount += 1;
|
reconcileCount += 1;
|
||||||
|
lifecycle.push('runtime:reconcile');
|
||||||
},
|
},
|
||||||
runScheduleNow: async () => {},
|
runScheduleNow: async () => {},
|
||||||
runBuildQueueNow: async () => {},
|
runBuildQueueNow: async () => {},
|
||||||
@@ -297,6 +300,13 @@ const buildCaller = async (
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
cleanupStaleWorkspaces: async () => ({ removed: [], skipped: [] }),
|
cleanupStaleWorkspaces: async () => ({ removed: [], skipped: [] }),
|
||||||
|
transitionProfileClock: async (_profileName, action) => {
|
||||||
|
lifecycle.push(`clock:${action}`);
|
||||||
|
return {
|
||||||
|
phase: action === 'SUSPEND' ? 'SUSPENDED' : 'RUNNING',
|
||||||
|
revision: 1,
|
||||||
|
};
|
||||||
|
},
|
||||||
listRuntimeSettings: async () => [
|
listRuntimeSettings: async () => [
|
||||||
{
|
{
|
||||||
profileName: 'che:2',
|
profileName: 'che:2',
|
||||||
@@ -363,6 +373,7 @@ const buildCaller = async (
|
|||||||
updatedStatuses,
|
updatedStatuses,
|
||||||
updatedMetas,
|
updatedMetas,
|
||||||
auditEvents,
|
auditEvents,
|
||||||
|
lifecycle,
|
||||||
getReconcileCount: () => reconcileCount,
|
getReconcileCount: () => reconcileCount,
|
||||||
getRuntimeStateListCount: () => runtimeStateListCount,
|
getRuntimeStateListCount: () => runtimeStateListCount,
|
||||||
getStoredNotice: () => storedNotice,
|
getStoredNotice: () => storedNotice,
|
||||||
@@ -1384,6 +1395,7 @@ describe('admin runtime clock action API', () => {
|
|||||||
})
|
})
|
||||||
).resolves.toMatchObject({ ok: true });
|
).resolves.toMatchObject({ ok: true });
|
||||||
expect(harness.updatedStatuses).toEqual(['RUNNING']);
|
expect(harness.updatedStatuses).toEqual(['RUNNING']);
|
||||||
|
expect(harness.lifecycle).toEqual(['clock:RESUME', 'status:RUNNING', 'runtime:reconcile']);
|
||||||
expect(harness.getReconcileCount()).toBe(1);
|
expect(harness.getReconcileCount()).toBe(1);
|
||||||
expect(harness.updatedMetas).toHaveLength(2);
|
expect(harness.updatedMetas).toHaveLength(2);
|
||||||
expect(harness.updatedMetas.at(-1)).toMatchObject({
|
expect(harness.updatedMetas.at(-1)).toMatchObject({
|
||||||
@@ -1410,6 +1422,11 @@ describe('admin runtime clock action API', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(harness.updatedStatuses).toEqual([expectedStatus]);
|
expect(harness.updatedStatuses).toEqual([expectedStatus]);
|
||||||
|
expect(harness.lifecycle).toEqual(
|
||||||
|
action === 'STOP'
|
||||||
|
? ['clock:SUSPEND', `status:${expectedStatus}`, 'runtime:reconcile']
|
||||||
|
: [`status:${expectedStatus}`, 'runtime:reconcile']
|
||||||
|
);
|
||||||
expect(harness.getReconcileCount()).toBe(1);
|
expect(harness.getReconcileCount()).toBe(1);
|
||||||
expect(harness.updatedMetas).toHaveLength(2);
|
expect(harness.updatedMetas).toHaveLength(2);
|
||||||
expect(harness.updatedMetas[0]).toMatchObject({
|
expect(harness.updatedMetas[0]).toMatchObject({
|
||||||
|
|||||||
@@ -123,6 +123,10 @@ const createHarness = async (adminRoles = ['user', 'admin.users.manage', 'admin.
|
|||||||
runBuildQueueNow: async () => {},
|
runBuildQueueNow: async () => {},
|
||||||
runOperationsNow: async () => {},
|
runOperationsNow: async () => {},
|
||||||
cleanupStaleWorkspaces: async () => ({ removed: [], skipped: [] }),
|
cleanupStaleWorkspaces: async () => ({ removed: [], skipped: [] }),
|
||||||
|
transitionProfileClock: async (_profileName, action) => ({
|
||||||
|
phase: action === 'SUSPEND' ? 'SUSPENDED' : 'RUNNING',
|
||||||
|
revision: 1,
|
||||||
|
}),
|
||||||
listRuntimeStates: async () => [],
|
listRuntimeStates: async () => [],
|
||||||
},
|
},
|
||||||
profileStatus: new InMemoryProfileStatusService(),
|
profileStatus: new InMemoryProfileStatusService(),
|
||||||
@@ -183,9 +187,7 @@ describe('admin security over HTTP transport', () => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const mutationInput = encodeURIComponent(
|
const mutationInput = encodeURIComponent(JSON.stringify({ json: { sessionToken: harness.adminSessionToken } }));
|
||||||
JSON.stringify({ json: { sessionToken: harness.adminSessionToken } })
|
|
||||||
);
|
|
||||||
const mutation = await fetch(`${harness.baseUrl}/trpc/auth.logout?input=${mutationInput}`);
|
const mutation = await fetch(`${harness.baseUrl}/trpc/auth.logout?input=${mutationInput}`);
|
||||||
expect(mutation.status).toBe(405);
|
expect(mutation.status).toBe(405);
|
||||||
expect(await mutation.json()).toMatchObject({
|
expect(await mutation.json()).toMatchObject({
|
||||||
|
|||||||
@@ -168,6 +168,10 @@ const buildCaller = (
|
|||||||
removed: [],
|
removed: [],
|
||||||
skipped: [],
|
skipped: [],
|
||||||
}),
|
}),
|
||||||
|
transitionProfileClock: async (_profileName: string, action: 'SUSPEND' | 'RESUME') => ({
|
||||||
|
phase: action === 'SUSPEND' ? 'SUSPENDED' : 'RUNNING',
|
||||||
|
revision: 1,
|
||||||
|
}),
|
||||||
listRuntimeStates: async () => [],
|
listRuntimeStates: async () => [],
|
||||||
};
|
};
|
||||||
const profileStatus = new InMemoryProfileStatusService(
|
const profileStatus = new InMemoryProfileStatusService(
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ const createHarness = (
|
|||||||
frontendServeMode?: 'static';
|
frontendServeMode?: 'static';
|
||||||
frontendArtifactRoot?: string;
|
frontendArtifactRoot?: string;
|
||||||
activeOperationProfileNames?: string[];
|
activeOperationProfileNames?: string[];
|
||||||
|
promoteProfileOpening?: GatewayOrchestratorOptions['promoteProfileOpening'];
|
||||||
} = {}
|
} = {}
|
||||||
) => {
|
) => {
|
||||||
const harnessProfile = options.profile ?? profile;
|
const harnessProfile = options.profile ?? profile;
|
||||||
@@ -77,6 +78,7 @@ const createHarness = (
|
|||||||
const deleted: string[] = [];
|
const deleted: string[] = [];
|
||||||
const buildStatuses: string[] = [];
|
const buildStatuses: string[] = [];
|
||||||
const logs: Array<{ phase: string; message: string; level: string }> = [];
|
const logs: Array<{ phase: string; message: string; level: string }> = [];
|
||||||
|
const lifecycle: string[] = [];
|
||||||
|
|
||||||
const repository: GatewayProfileRepository = {
|
const repository: GatewayProfileRepository = {
|
||||||
listProfiles: async () => options.profiles ?? [harnessProfile],
|
listProfiles: async () => options.profiles ?? [harnessProfile],
|
||||||
@@ -85,6 +87,7 @@ const createHarness = (
|
|||||||
updateCurrentScenario: async () => harnessProfile,
|
updateCurrentScenario: async () => harnessProfile,
|
||||||
updateStatus: async (_profileName, status) => {
|
updateStatus: async (_profileName, status) => {
|
||||||
statuses.push(status);
|
statuses.push(status);
|
||||||
|
lifecycle.push(`status:${status}`);
|
||||||
return { ...harnessProfile, status };
|
return { ...harnessProfile, status };
|
||||||
},
|
},
|
||||||
updateBuildStatus: async (_profileName, status) => {
|
updateBuildStatus: async (_profileName, status) => {
|
||||||
@@ -192,9 +195,26 @@ const createHarness = (
|
|||||||
adminActionIntervalMs: 60_000,
|
adminActionIntervalMs: 60_000,
|
||||||
now: options.now,
|
now: options.now,
|
||||||
cancelGame: options.cancelGame,
|
cancelGame: options.cancelGame,
|
||||||
|
promoteProfileOpening: options.promoteProfileOpening
|
||||||
|
? async (openingProfile) => {
|
||||||
|
lifecycle.push(`clock:${openingProfile.profileName}`);
|
||||||
|
await options.promoteProfileOpening?.(openingProfile);
|
||||||
|
}
|
||||||
|
: undefined,
|
||||||
});
|
});
|
||||||
|
|
||||||
return { orchestrator, statuses, buildStatuses, completions, completionFields, started, stopped, deleted, logs };
|
return {
|
||||||
|
orchestrator,
|
||||||
|
statuses,
|
||||||
|
buildStatuses,
|
||||||
|
completions,
|
||||||
|
completionFields,
|
||||||
|
started,
|
||||||
|
stopped,
|
||||||
|
deleted,
|
||||||
|
logs,
|
||||||
|
lifecycle,
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
describe('GatewayOrchestrator first-class operations', () => {
|
describe('GatewayOrchestrator first-class operations', () => {
|
||||||
@@ -333,12 +353,14 @@ describe('GatewayOrchestrator first-class operations', () => {
|
|||||||
profiles: [],
|
profiles: [],
|
||||||
reservedToStart: [reservedProfile],
|
reservedToStart: [reservedProfile],
|
||||||
now: () => now,
|
now: () => now,
|
||||||
|
promoteProfileOpening: async () => {},
|
||||||
});
|
});
|
||||||
|
|
||||||
await harness.orchestrator.runScheduleNow();
|
await harness.orchestrator.runScheduleNow();
|
||||||
|
|
||||||
expect(harness.statuses).toEqual(['RUNNING']);
|
expect(harness.statuses).toEqual(['RUNNING']);
|
||||||
expect(harness.buildStatuses).toEqual([]);
|
expect(harness.buildStatuses).toEqual([]);
|
||||||
|
expect(harness.lifecycle).toEqual([`clock:${reservedProfile.profileName}`, 'status:RUNNING']);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('retains the legacy build queue for an unprepared reserved profile', async () => {
|
it('retains the legacy build queue for an unprepared reserved profile', async () => {
|
||||||
@@ -366,6 +388,26 @@ describe('GatewayOrchestrator first-class operations', () => {
|
|||||||
expect(harness.buildStatuses).toEqual(['QUEUED']);
|
expect(harness.buildStatuses).toEqual(['QUEUED']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('promotes the durable clock before a prepared preopen profile becomes running', async () => {
|
||||||
|
const now = new Date('2030-01-01T02:00:00.000Z');
|
||||||
|
const preopenProfile: GatewayProfileRecord = {
|
||||||
|
...profile,
|
||||||
|
status: 'PREOPEN',
|
||||||
|
openAt: now.toISOString(),
|
||||||
|
preopenAt: '2030-01-01T01:00:00.000Z',
|
||||||
|
};
|
||||||
|
const harness = createHarness(buildOperation('START'), false, false, false, false, undefined, undefined, {
|
||||||
|
profile: preopenProfile,
|
||||||
|
profiles: [preopenProfile],
|
||||||
|
now: () => now,
|
||||||
|
promoteProfileOpening: async () => {},
|
||||||
|
});
|
||||||
|
|
||||||
|
await harness.orchestrator.runScheduleNow();
|
||||||
|
|
||||||
|
expect(harness.lifecycle).toEqual([`clock:${preopenProfile.profileName}`, 'status:RUNNING']);
|
||||||
|
});
|
||||||
|
|
||||||
it('starts every profile process and records success', async () => {
|
it('starts every profile process and records success', async () => {
|
||||||
const harness = createHarness(buildOperation('START'));
|
const harness = createHarness(buildOperation('START'));
|
||||||
|
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ mean deployment or production validation.
|
|||||||
|
|
||||||
- [x] All durable input events record accepted tick and accepted revision.
|
- [x] All durable input events record accepted tick and accepted revision.
|
||||||
- [x] Processing converts accepted coordinates across revisions or fails closed.
|
- [x] Processing converts accepted coordinates across revisions or fails closed.
|
||||||
- [ ] Gateway pause/resume/open orchestration writes the DB clock phase.
|
- [x] Gateway pause/resume/open orchestration writes the DB clock phase.
|
||||||
- [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension.
|
- [ ] Unification wait becomes a durable `UNIFICATION_WAIT` suspension.
|
||||||
- [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule,
|
- [ ] Alignment, optional rate change, invader IDs/RNG, creation, first schedule,
|
||||||
outbox, verification, and RUNNING transition form one retry-safe workflow.
|
outbox, verification, and RUNNING transition form one retry-safe workflow.
|
||||||
@@ -101,3 +101,14 @@ mean deployment or production validation.
|
|||||||
- The 24-hour/65m17.250s reconciliation suite passed 2/2 after the other DB
|
- The 24-hour/65m17.250s reconciliation suite passed 2/2 after the other DB
|
||||||
suites. Conditional files share a deliberately dedicated schema and are run
|
suites. Conditional files share a deliberately dedicated schema and are run
|
||||||
sequentially to prevent their fixture cleanup from racing another file.
|
sequentially to prevent their fixture cleanup from racing another file.
|
||||||
|
|
||||||
|
### 2026-09-03 - Gateway lifecycle authority
|
||||||
|
|
||||||
|
- Runtime `PAUSE`/`STOP` starts a durable maintenance suspension before the
|
||||||
|
Gateway status and process reconciliation change. `RESUME` completes the DB
|
||||||
|
reconciliation and Redis outbox before the profile becomes `RUNNING`.
|
||||||
|
- Both an already-built overdue `RESERVED` profile and an overdue `PREOPEN`
|
||||||
|
profile promote the game DB from `PREOPEN@0` before the Gateway status changes
|
||||||
|
to `RUNNING`; the Redis clock phase is revision/generation fenced.
|
||||||
|
- `pnpm --filter @sammo-ts/gateway-api test` passed 313 tests with 35
|
||||||
|
environment-conditional skips. Gateway typecheck and target lint passed.
|
||||||
|
|||||||
Reference in New Issue
Block a user