feat: 레거시 DB 순차 증분 이관 CLI 추가
This commit is contained in:
@@ -9,8 +9,11 @@ import { isLegacyArchiveProfile, LEGACY_ARCHIVE_PROFILES, migrateGame } from './
|
||||
import { migrateGateway } from './gateway.js';
|
||||
import { hashPasswordForReset } from './password.js';
|
||||
import { migrateCurrentSeasonFixture } from './currentSeason.js';
|
||||
import { loadMigrationPlan } from './config.js';
|
||||
import { fingerprintMariaConnection, type MigrationMode } from './incremental.js';
|
||||
import { checkMigrationPlan, runMigrationPlan } from './plan.js';
|
||||
|
||||
type Command = 'gateway' | 'game' | 'current-season-fixture' | 'reset-password';
|
||||
type Command = 'gateway' | 'game' | 'current-season-fixture' | 'reset-password' | 'check-plan' | 'run-plan';
|
||||
|
||||
interface CliOptions {
|
||||
command: Command;
|
||||
@@ -22,11 +25,17 @@ interface CliOptions {
|
||||
expectedYear?: number;
|
||||
expectedMonth?: number;
|
||||
replaceCurrentSeason: boolean;
|
||||
config?: string;
|
||||
mode: MigrationMode;
|
||||
sourceKey?: string;
|
||||
}
|
||||
|
||||
const usage = `Usage:
|
||||
pnpm --filter @sammo-ts/legacy-db-migration migrate gateway [--apply]
|
||||
pnpm --filter @sammo-ts/legacy-db-migration migrate game --profile <profile> [--apply]
|
||||
pnpm --filter @sammo-ts/legacy-db-migration migrate check-plan --config <secure-plan.json>
|
||||
pnpm --filter @sammo-ts/legacy-db-migration migrate run-plan --config <secure-plan.json> \
|
||||
[--mode full|incremental] [--apply]
|
||||
pnpm --filter @sammo-ts/legacy-db-migration migrate current-season-fixture --profile <profile> \
|
||||
--expected-scenario <id> --expected-year <year> --expected-month <month> \
|
||||
[--replace-current-season --apply]
|
||||
@@ -47,11 +56,13 @@ const parseArguments = (argv: readonly string[]): CliOptions => {
|
||||
command !== 'gateway' &&
|
||||
command !== 'game' &&
|
||||
command !== 'current-season-fixture' &&
|
||||
command !== 'reset-password'
|
||||
command !== 'reset-password' &&
|
||||
command !== 'check-plan' &&
|
||||
command !== 'run-plan'
|
||||
) {
|
||||
throw new Error(usage);
|
||||
}
|
||||
const options: CliOptions = { command, apply: false, replaceCurrentSeason: false };
|
||||
const options: CliOptions = { command, apply: false, replaceCurrentSeason: false, mode: 'full' };
|
||||
for (let index = 1; index < argv.length; index += 1) {
|
||||
const argument = argv[index];
|
||||
if (argument === '--apply') {
|
||||
@@ -68,6 +79,15 @@ const parseArguments = (argv: readonly string[]): CliOptions => {
|
||||
}
|
||||
if (argument === '--profile') {
|
||||
options.profile = next;
|
||||
} else if (argument === '--config') {
|
||||
options.config = next;
|
||||
} else if (argument === '--mode') {
|
||||
if (next !== 'full' && next !== 'incremental') {
|
||||
throw new Error(`--mode must be full or incremental\n\n${usage}`);
|
||||
}
|
||||
options.mode = next;
|
||||
} else if (argument === '--source-key') {
|
||||
options.sourceKey = next;
|
||||
} else if (argument === '--login-id') {
|
||||
options.loginId = next;
|
||||
} else if (argument === '--password-file') {
|
||||
@@ -94,6 +114,9 @@ const requireEnvironment = (name: string): string => {
|
||||
return value;
|
||||
};
|
||||
|
||||
const resolveInvocationPath = (value: string): string =>
|
||||
path.resolve(process.env.INIT_CWD?.trim() || process.cwd(), value);
|
||||
|
||||
const resetPassword = async (options: CliOptions): Promise<Record<string, unknown>> => {
|
||||
if (!options.apply) {
|
||||
throw new Error('reset-password requires --apply');
|
||||
@@ -101,7 +124,7 @@ const resetPassword = async (options: CliOptions): Promise<Record<string, unknow
|
||||
if (!options.loginId || !options.passwordFile) {
|
||||
throw new Error(`reset-password requires --login-id and --password-file\n\n${usage}`);
|
||||
}
|
||||
const passwordPath = path.resolve(options.passwordFile);
|
||||
const passwordPath = resolveInvocationPath(options.passwordFile);
|
||||
const passwordStat = await stat(passwordPath);
|
||||
if ((passwordStat.mode & 0o077) !== 0) {
|
||||
throw new Error('Password file must not be readable or writable by group/other (expected mode 0600)');
|
||||
@@ -136,7 +159,23 @@ const resetPassword = async (options: CliOptions): Promise<Record<string, unknow
|
||||
};
|
||||
|
||||
const run = async (): Promise<void> => {
|
||||
const options = parseArguments(process.argv.slice(2));
|
||||
const argumentsAfterScript = process.argv.slice(2);
|
||||
if (argumentsAfterScript[0] === '--') argumentsAfterScript.shift();
|
||||
if (argumentsAfterScript[0] === '--help' || argumentsAfterScript[0] === '-h') {
|
||||
console.log(usage);
|
||||
return;
|
||||
}
|
||||
const options = parseArguments(argumentsAfterScript);
|
||||
if (options.command === 'check-plan' || options.command === 'run-plan') {
|
||||
if (!options.config) throw new Error(`${options.command} requires --config\n\n${usage}`);
|
||||
const plan = await loadMigrationPlan(resolveInvocationPath(options.config));
|
||||
const result =
|
||||
options.command === 'check-plan'
|
||||
? await checkMigrationPlan(plan)
|
||||
: await runMigrationPlan(plan, options.mode, options.apply);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
if (options.command === 'reset-password') {
|
||||
console.log(JSON.stringify(await resetPassword(options), null, 2));
|
||||
return;
|
||||
@@ -144,14 +183,21 @@ const run = async (): Promise<void> => {
|
||||
|
||||
const migratedAt = new Date();
|
||||
if (options.command === 'gateway') {
|
||||
const source = createMariaPool(requireEnvironment('LEGACY_ROOT_DATABASE_URL'));
|
||||
const sourceUrl = requireEnvironment('LEGACY_ROOT_DATABASE_URL');
|
||||
const source = createMariaPool(sourceUrl);
|
||||
const targetUrl = process.env.GATEWAY_DATABASE_URL?.trim();
|
||||
if (options.apply && !targetUrl) {
|
||||
throw new Error('GATEWAY_DATABASE_URL is required with --apply');
|
||||
}
|
||||
const target = targetUrl ? createPostgresPool(targetUrl) : null;
|
||||
try {
|
||||
const summary = await migrateGateway(source, target, options.apply, migratedAt);
|
||||
const summary = await migrateGateway(source, target, options.apply, migratedAt, {
|
||||
mode: options.mode,
|
||||
source: {
|
||||
key: options.sourceKey ?? process.env.LEGACY_SOURCE_KEY?.trim() ?? 'legacy-root',
|
||||
fingerprint: fingerprintMariaConnection(sourceUrl),
|
||||
},
|
||||
});
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
} finally {
|
||||
await source.end();
|
||||
@@ -166,9 +212,10 @@ const run = async (): Promise<void> => {
|
||||
if (options.command === 'game' && !isLegacyArchiveProfile(options.profile)) {
|
||||
throw new Error(`game requires --profile ${LEGACY_ARCHIVE_PROFILES.join('|')}\n\n${usage}`);
|
||||
}
|
||||
const source = createMariaPool(requireEnvironment('LEGACY_GAME_DATABASE_URL'));
|
||||
const sourceUrl = requireEnvironment('LEGACY_GAME_DATABASE_URL');
|
||||
const source = createMariaPool(sourceUrl);
|
||||
const target =
|
||||
options.apply || options.command === 'current-season-fixture'
|
||||
options.apply || options.mode === 'incremental' || options.command === 'current-season-fixture'
|
||||
? createPostgresPool(requireEnvironment('GAME_DATABASE_URL'))
|
||||
: null;
|
||||
try {
|
||||
@@ -199,7 +246,13 @@ const run = async (): Promise<void> => {
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
return;
|
||||
}
|
||||
const summary = await migrateGame(source, target, options.apply, options.profile);
|
||||
const summary = await migrateGame(source, target, options.apply, options.profile, {
|
||||
mode: options.mode,
|
||||
source: {
|
||||
key: options.sourceKey ?? process.env.LEGACY_SOURCE_KEY?.trim() ?? `legacy-game-${options.profile}`,
|
||||
fingerprint: fingerprintMariaConnection(sourceUrl),
|
||||
},
|
||||
});
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
} finally {
|
||||
await source.end();
|
||||
@@ -208,7 +261,9 @@ const run = async (): Promise<void> => {
|
||||
};
|
||||
|
||||
run().catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const message = (error instanceof Error ? error.message : String(error))
|
||||
.replace(/((?:mariadb|mysql|postgres(?:ql)?):\/\/[^:\s/@]+:)[^@\s/]+@/giu, '$1***@')
|
||||
.replace(/([?&](?:pass(?:word)?|secret|token)=)[^&\s]+/giu, '$1***');
|
||||
console.error(`[legacy-db-migration] ${message}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
import { constants } from 'node:fs';
|
||||
import { lstat, open } from 'node:fs/promises';
|
||||
import { isIP } from 'node:net';
|
||||
import path from 'node:path';
|
||||
|
||||
import { isLegacyArchiveProfile, LEGACY_ARCHIVE_PROFILES, type LegacyArchiveProfile } from './game.js';
|
||||
import { fingerprintMariaConnection, type MigrationSourceIdentity } from './incremental.js';
|
||||
|
||||
export interface ResolvedMigrationStage {
|
||||
kind: 'gateway' | 'game';
|
||||
name: string;
|
||||
profile?: LegacyArchiveProfile;
|
||||
sourceUrl: string;
|
||||
targetUrl: string;
|
||||
sourceIdentity: MigrationSourceIdentity;
|
||||
}
|
||||
|
||||
export interface ResolvedMigrationPlan {
|
||||
sourceSet: string;
|
||||
stages: ResolvedMigrationStage[];
|
||||
}
|
||||
|
||||
const SAFE_KEY = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,95}$/u;
|
||||
const ENV_NAME = /^[A-Z][A-Z0-9_]{1,127}$/u;
|
||||
|
||||
const assertSecureRegularFile = async (filePath: string, label: string): Promise<void> => {
|
||||
const info = await lstat(filePath);
|
||||
if (!info.isFile() || info.isSymbolicLink()) {
|
||||
throw new Error(`${label} must be a regular file and not a symbolic link`);
|
||||
}
|
||||
if ((info.mode & 0o077) !== 0) {
|
||||
throw new Error(`${label} must not be readable or writable by group/other (expected mode 0600)`);
|
||||
}
|
||||
};
|
||||
|
||||
const readSecureText = async (filePath: string, label: string): Promise<string> => {
|
||||
await assertSecureRegularFile(filePath, label);
|
||||
const handle = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
return await handle.readFile('utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
|
||||
const asRecord = (value: unknown, label: string): Record<string, unknown> => {
|
||||
if (value === null || Array.isArray(value) || typeof value !== 'object') {
|
||||
throw new Error(`${label} must be an object`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
};
|
||||
|
||||
const requiredString = (record: Record<string, unknown>, key: string, label: string): string => {
|
||||
const value = record[key];
|
||||
if (typeof value !== 'string' || !value.trim()) {
|
||||
throw new Error(`${label}.${key} must be a non-empty string`);
|
||||
}
|
||||
return value.trim();
|
||||
};
|
||||
|
||||
const rejectUnknownKeys = (record: Record<string, unknown>, allowed: readonly string[], label: string): void => {
|
||||
const unknown = Object.keys(record).filter((key) => !allowed.includes(key));
|
||||
if (unknown.length) {
|
||||
throw new Error(`${label} has unknown keys: ${unknown.join(', ')}`);
|
||||
}
|
||||
};
|
||||
|
||||
const resolvePassword = async (
|
||||
source: Record<string, unknown>,
|
||||
configDirectory: string,
|
||||
label: string
|
||||
): Promise<string> => {
|
||||
const configured = ['password', 'passwordEnv', 'passwordFile'].filter(
|
||||
(key) => typeof source[key] === 'string' && Boolean(String(source[key]).trim())
|
||||
);
|
||||
if (configured.length !== 1) {
|
||||
throw new Error(`${label} must configure exactly one of password, passwordEnv, or passwordFile`);
|
||||
}
|
||||
if (configured[0] === 'password') {
|
||||
return requiredString(source, 'password', label);
|
||||
}
|
||||
if (configured[0] === 'passwordEnv') {
|
||||
const environmentName = requiredString(source, 'passwordEnv', label);
|
||||
if (!ENV_NAME.test(environmentName)) throw new Error(`${label}.passwordEnv is not a safe environment name`);
|
||||
const value = process.env[environmentName];
|
||||
if (!value) throw new Error(`${environmentName} is required by ${label}`);
|
||||
return value;
|
||||
}
|
||||
const configuredPath = requiredString(source, 'passwordFile', label);
|
||||
const passwordPath = path.resolve(configDirectory, configuredPath);
|
||||
const value = (await readSecureText(passwordPath, `${label}.passwordFile`)).replace(/\r?\n$/u, '');
|
||||
if (!value) throw new Error(`${label}.passwordFile is empty`);
|
||||
return value;
|
||||
};
|
||||
|
||||
const resolveSource = async (value: unknown, configDirectory: string, label: string): Promise<string> => {
|
||||
const source = asRecord(value, label);
|
||||
rejectUnknownKeys(
|
||||
source,
|
||||
['host', 'port', 'database', 'user', 'password', 'passwordEnv', 'passwordFile', 'tls'],
|
||||
label
|
||||
);
|
||||
const host = requiredString(source, 'host', label);
|
||||
const database = requiredString(source, 'database', label);
|
||||
const user = requiredString(source, 'user', label);
|
||||
const port = source.port === undefined ? 3306 : Number(source.port);
|
||||
if (!Number.isSafeInteger(port) || port < 1 || port > 65535) {
|
||||
throw new Error(`${label}.port must be an integer from 1 through 65535`);
|
||||
}
|
||||
const dnsName =
|
||||
host.length <= 253 && host.split('.').every((part) => /^(?!-)[a-zA-Z0-9-]{1,63}(?<!-)$/u.test(part));
|
||||
if (!isIP(host) && !dnsName) throw new Error(`${label}.host must be an IP address or DNS name`);
|
||||
if (!/^[a-zA-Z0-9_$.-]{1,64}$/u.test(database)) {
|
||||
throw new Error(`${label}.database must be a safe MariaDB database name`);
|
||||
}
|
||||
if (source.tls !== undefined && typeof source.tls !== 'boolean') {
|
||||
throw new Error(`${label}.tls must be a boolean`);
|
||||
}
|
||||
const password = await resolvePassword(source, configDirectory, label);
|
||||
const url = new URL('mariadb://localhost');
|
||||
url.hostname = host;
|
||||
url.port = String(port);
|
||||
url.username = user;
|
||||
url.password = password;
|
||||
url.pathname = `/${database}`;
|
||||
if (source.tls) url.searchParams.set('ssl', 'true');
|
||||
return url.toString();
|
||||
};
|
||||
|
||||
const resolveTargetUrl = (record: Record<string, unknown>, label: string): string => {
|
||||
const environmentName = requiredString(record, 'targetUrlEnv', label);
|
||||
if (!ENV_NAME.test(environmentName)) throw new Error(`${label}.targetUrlEnv is not a safe environment name`);
|
||||
const value = process.env[environmentName]?.trim();
|
||||
if (!value) throw new Error(`${environmentName} is required by ${label}`);
|
||||
return value;
|
||||
};
|
||||
|
||||
const parseStage = (value: unknown, label: string): Record<string, unknown> => {
|
||||
const record = asRecord(value, label);
|
||||
rejectUnknownKeys(record, ['source', 'targetUrlEnv', 'profile', 'enabled'], label);
|
||||
if (!('source' in record)) throw new Error(`${label}.source is required`);
|
||||
return record;
|
||||
};
|
||||
|
||||
export const loadMigrationPlan = async (configPathInput: string): Promise<ResolvedMigrationPlan> => {
|
||||
const configPath = path.resolve(configPathInput);
|
||||
const rawText = await readSecureText(configPath, 'Migration config');
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(rawText);
|
||||
} catch (error) {
|
||||
throw new Error('Migration config is not valid JSON', { cause: error });
|
||||
}
|
||||
const root = asRecord(parsed, 'Migration config');
|
||||
rejectUnknownKeys(root, ['version', 'sourceSet', 'gateway', 'profiles'], 'Migration config');
|
||||
if (root.version !== 1) throw new Error('Migration config.version must be 1');
|
||||
const sourceSet = requiredString(root, 'sourceSet', 'Migration config');
|
||||
if (!SAFE_KEY.test(sourceSet)) throw new Error('Migration config.sourceSet must use safe characters');
|
||||
const configDirectory = path.dirname(configPath);
|
||||
const stages: ResolvedMigrationStage[] = [];
|
||||
|
||||
if (root.gateway !== undefined) {
|
||||
const gateway = parseStage(root.gateway, 'gateway');
|
||||
const sourceUrl = await resolveSource(gateway.source, configDirectory, 'gateway.source');
|
||||
stages.push({
|
||||
kind: 'gateway',
|
||||
name: 'gateway',
|
||||
sourceUrl,
|
||||
targetUrl: resolveTargetUrl(gateway, 'gateway'),
|
||||
sourceIdentity: {
|
||||
key: `${sourceSet}:gateway`,
|
||||
fingerprint: fingerprintMariaConnection(sourceUrl),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const profiles = root.profiles === undefined ? [] : root.profiles;
|
||||
if (!Array.isArray(profiles)) throw new Error('Migration config.profiles must be an array');
|
||||
const seen = new Set<string>();
|
||||
const profileStages = new Map<LegacyArchiveProfile, ResolvedMigrationStage>();
|
||||
for (const [index, value] of profiles.entries()) {
|
||||
const label = `profiles[${index}]`;
|
||||
const profileConfig = parseStage(value, label);
|
||||
if (profileConfig.enabled === false) continue;
|
||||
if (profileConfig.enabled !== undefined && profileConfig.enabled !== true) {
|
||||
throw new Error(`${label}.enabled must be a boolean`);
|
||||
}
|
||||
const profile = requiredString(profileConfig, 'profile', label);
|
||||
if (!isLegacyArchiveProfile(profile)) {
|
||||
throw new Error(`${label}.profile must be one of ${LEGACY_ARCHIVE_PROFILES.join(', ')}`);
|
||||
}
|
||||
if (seen.has(profile)) throw new Error(`Duplicate profile in migration config: ${profile}`);
|
||||
seen.add(profile);
|
||||
const sourceUrl = await resolveSource(profileConfig.source, configDirectory, `${label}.source`);
|
||||
profileStages.set(profile, {
|
||||
kind: 'game',
|
||||
name: profile,
|
||||
profile,
|
||||
sourceUrl,
|
||||
targetUrl: resolveTargetUrl(profileConfig, label),
|
||||
sourceIdentity: {
|
||||
key: `${sourceSet}:${profile}`,
|
||||
fingerprint: fingerprintMariaConnection(sourceUrl),
|
||||
},
|
||||
});
|
||||
}
|
||||
for (const profile of LEGACY_ARCHIVE_PROFILES) {
|
||||
const stage = profileStages.get(profile);
|
||||
if (stage) stages.push(stage);
|
||||
}
|
||||
if (!stages.length) throw new Error('Migration config has no enabled stages');
|
||||
return { sourceSet, stages };
|
||||
};
|
||||
|
||||
export const readPasswordFileForReset = async (passwordPath: string): Promise<string> => {
|
||||
const value = (await readSecureText(path.resolve(passwordPath), 'Password file')).replace(/\r?\n$/u, '');
|
||||
if (!value) throw new Error('Password file is empty');
|
||||
return value;
|
||||
};
|
||||
@@ -68,12 +68,13 @@ export const paginateSource = async function* (
|
||||
pool: MariaPool,
|
||||
table: string,
|
||||
idColumn: string,
|
||||
batchSize: number
|
||||
batchSize: number,
|
||||
afterId: bigint = -1n
|
||||
): AsyncGenerator<SourceRow[]> {
|
||||
if (!MARIA_IDENTIFIER.test(table) || !MARIA_IDENTIFIER.test(idColumn)) {
|
||||
throw new Error('Unsafe MariaDB table or ID column');
|
||||
}
|
||||
let lastId = -1n;
|
||||
let lastId = afterId;
|
||||
for (;;) {
|
||||
const rows = await querySource(
|
||||
pool,
|
||||
@@ -88,6 +89,15 @@ export const paginateSource = async function* (
|
||||
}
|
||||
};
|
||||
|
||||
export const sourceMaxId = async (pool: MariaPool, table: string, idColumn: string): Promise<bigint | null> => {
|
||||
if (!MARIA_IDENTIFIER.test(table) || !MARIA_IDENTIFIER.test(idColumn)) {
|
||||
throw new Error('Unsafe MariaDB table or ID column');
|
||||
}
|
||||
const rows = await querySource(pool, `SELECT MAX(\`${idColumn}\`) AS max_id FROM \`${table}\``);
|
||||
const value = rows[0]?.max_id;
|
||||
return value === null || value === undefined ? null : toBigInt(value, `${table}.${idColumn}`);
|
||||
};
|
||||
|
||||
const targetValue = (value: unknown): unknown => {
|
||||
if (value instanceof JsonParameter) {
|
||||
return JSON.stringify(value.value);
|
||||
|
||||
@@ -16,6 +16,7 @@ export { isLegacyArchiveProfile, LEGACY_ARCHIVE_PROFILES, type LegacyArchiveProf
|
||||
import {
|
||||
paginateSource,
|
||||
jsonParameter,
|
||||
sourceMaxId,
|
||||
toDate,
|
||||
toFloat,
|
||||
toNullableDate,
|
||||
@@ -28,6 +29,15 @@ import {
|
||||
type TargetRow,
|
||||
} from './db.js';
|
||||
import type { MigrationSummary } from './gateway.js';
|
||||
import {
|
||||
defaultExecutionOptions,
|
||||
loadCheckpoint,
|
||||
requireIncrementalCheckpoint,
|
||||
saveCheckpoint,
|
||||
validateSourceIdentity,
|
||||
type MigrationExecutionOptions,
|
||||
type MigrationProgress,
|
||||
} from './incremental.js';
|
||||
import { legacyUserId } from './identity.js';
|
||||
import {
|
||||
classifyGameStorage,
|
||||
@@ -45,6 +55,11 @@ interface ArchiveMigrationContext {
|
||||
sourceFormats: Record<ArchivedGeneralSourceFormat, number>;
|
||||
}
|
||||
|
||||
interface AppendCursor {
|
||||
afterId: bigint;
|
||||
endAtId: bigint;
|
||||
}
|
||||
|
||||
const parseNullableJson = (value: unknown, fallback: JsonValue, context: string): JsonValue =>
|
||||
value === null || value === undefined ? fallback : parseJson(value, context);
|
||||
|
||||
@@ -80,22 +95,33 @@ const migrateSimpleTable = async (
|
||||
conflictColumns: readonly string[],
|
||||
mapper: (row: SourceRow) => TargetRow,
|
||||
counts: Record<string, number>,
|
||||
size = batchSize
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor,
|
||||
size = batchSize,
|
||||
strategy: 'append' | 'rescan' = 'append'
|
||||
): Promise<void> => {
|
||||
for await (const rows of paginateSource(source, sourceTable, sourceIdColumn, size)) {
|
||||
for await (const rows of paginateSource(source, sourceTable, sourceIdColumn, size, cursor.afterId)) {
|
||||
const mapped = rows.map(mapper);
|
||||
if (target) {
|
||||
await upsertRows(target, targetTable, mapped, conflictColumns);
|
||||
}
|
||||
counts[sourceTable] = (counts[sourceTable] ?? 0) + mapped.length;
|
||||
}
|
||||
progress[sourceTable] = {
|
||||
strategy,
|
||||
startAfterId: cursor.afterId < 0n ? null : cursor.afterId.toString(),
|
||||
endAtId: cursor.endAtId < 0n ? null : cursor.endAtId.toString(),
|
||||
processed: counts[sourceTable] ?? 0,
|
||||
};
|
||||
};
|
||||
|
||||
const migrateHall = (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>,
|
||||
archive: ArchiveMigrationContext
|
||||
archive: ArchiveMigrationContext,
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor
|
||||
): Promise<void> =>
|
||||
migrateSimpleTable(
|
||||
source,
|
||||
@@ -120,14 +146,18 @@ const migrateHall = (
|
||||
import_run_id: archive.importRunId,
|
||||
};
|
||||
},
|
||||
counts
|
||||
counts,
|
||||
progress,
|
||||
cursor
|
||||
);
|
||||
|
||||
const migrateGames = (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>,
|
||||
archive: ArchiveMigrationContext
|
||||
archive: ArchiveMigrationContext,
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor
|
||||
): Promise<void> =>
|
||||
migrateSimpleTable(
|
||||
source,
|
||||
@@ -159,14 +189,20 @@ const migrateGames = (
|
||||
import_run_id: archive.importRunId,
|
||||
};
|
||||
},
|
||||
counts
|
||||
counts,
|
||||
progress,
|
||||
cursor,
|
||||
batchSize,
|
||||
'rescan'
|
||||
);
|
||||
|
||||
const migrateOldGenerals = (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>,
|
||||
archive: ArchiveMigrationContext
|
||||
archive: ArchiveMigrationContext,
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor
|
||||
): Promise<void> =>
|
||||
migrateSimpleTable(
|
||||
source,
|
||||
@@ -197,14 +233,18 @@ const migrateOldGenerals = (
|
||||
import_run_id: archive.importRunId,
|
||||
};
|
||||
},
|
||||
counts
|
||||
counts,
|
||||
progress,
|
||||
cursor
|
||||
);
|
||||
|
||||
const migrateOldNations = (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>,
|
||||
archive: ArchiveMigrationContext
|
||||
archive: ArchiveMigrationContext,
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor
|
||||
): Promise<void> =>
|
||||
migrateSimpleTable(
|
||||
source,
|
||||
@@ -225,14 +265,18 @@ const migrateOldNations = (
|
||||
import_run_id: archive.importRunId,
|
||||
};
|
||||
},
|
||||
counts
|
||||
counts,
|
||||
progress,
|
||||
cursor
|
||||
);
|
||||
|
||||
const migrateEmperors = (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>,
|
||||
archive: ArchiveMigrationContext
|
||||
archive: ArchiveMigrationContext,
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor
|
||||
): Promise<void> =>
|
||||
migrateSimpleTable(
|
||||
source,
|
||||
@@ -293,13 +337,17 @@ const migrateEmperors = (
|
||||
import_run_id: archive.importRunId,
|
||||
};
|
||||
},
|
||||
counts
|
||||
counts,
|
||||
progress,
|
||||
cursor
|
||||
);
|
||||
|
||||
const migrateInheritanceResults = (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>
|
||||
counts: Record<string, number>,
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor
|
||||
): Promise<void> =>
|
||||
migrateSimpleTable(
|
||||
source,
|
||||
@@ -321,13 +369,17 @@ const migrateInheritanceResults = (
|
||||
created_at: new Date(0),
|
||||
};
|
||||
},
|
||||
counts
|
||||
counts,
|
||||
progress,
|
||||
cursor
|
||||
);
|
||||
|
||||
const migrateUserRecords = (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>
|
||||
counts: Record<string, number>,
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor
|
||||
): Promise<void> =>
|
||||
migrateSimpleTable(
|
||||
source,
|
||||
@@ -349,14 +401,18 @@ const migrateUserRecords = (
|
||||
created_at: toNullableDate(row.date, `user_record.${id}.date`) ?? new Date(0),
|
||||
};
|
||||
},
|
||||
counts
|
||||
counts,
|
||||
progress,
|
||||
cursor
|
||||
);
|
||||
|
||||
const migrateYearbook = async (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>,
|
||||
archive: ArchiveMigrationContext
|
||||
archive: ArchiveMigrationContext,
|
||||
progress: MigrationProgress,
|
||||
cursor: AppendCursor
|
||||
): Promise<void> => {
|
||||
await migrateSimpleTable(
|
||||
source,
|
||||
@@ -387,6 +443,8 @@ const migrateYearbook = async (
|
||||
return mapped;
|
||||
},
|
||||
counts,
|
||||
progress,
|
||||
cursor,
|
||||
25
|
||||
);
|
||||
};
|
||||
@@ -451,12 +509,18 @@ export const migrateGame = async (
|
||||
source: MariaPool,
|
||||
targetPool: PgPool | null,
|
||||
apply: boolean,
|
||||
profile: string
|
||||
profile: string,
|
||||
execution: MigrationExecutionOptions = defaultExecutionOptions(`legacy-game-${profile}`)
|
||||
): Promise<MigrationSummary> => {
|
||||
if (!isLegacyArchiveProfile(profile)) {
|
||||
throw new Error(`Unsupported legacy archive profile: ${profile}`);
|
||||
}
|
||||
validateSourceIdentity(execution.source);
|
||||
if (execution.mode === 'incremental' && !targetPool) {
|
||||
throw new Error('Incremental game migration requires the target database to read checkpoints');
|
||||
}
|
||||
const counts: Record<string, number> = {};
|
||||
const progress: MigrationProgress = {};
|
||||
const sourceFormats: Record<ArchivedGeneralSourceFormat, number> = {
|
||||
'legacy-flat-v0': 0,
|
||||
'ref-flat-v1': 0,
|
||||
@@ -494,26 +558,88 @@ export const migrateGame = async (
|
||||
vote_comment: 'Current-season vote comments.',
|
||||
'storage:season-state': 'Only inheritance_* and user_* long-lived namespaces are archived or projected.',
|
||||
};
|
||||
const client = apply && targetPool ? await targetPool.connect() : null;
|
||||
const client = targetPool ? await targetPool.connect() : null;
|
||||
let importRunId: string | null = null;
|
||||
try {
|
||||
const run = async (archive: ArchiveMigrationContext): Promise<void> => {
|
||||
await migrateGames(source, client, counts, archive);
|
||||
await migrateHall(source, client, counts, archive);
|
||||
await migrateOldGenerals(source, client, counts, archive);
|
||||
await migrateOldNations(source, client, counts, archive);
|
||||
await migrateEmperors(source, client, counts, archive);
|
||||
await migrateInheritanceResults(source, client, counts);
|
||||
await migrateUserRecords(source, client, counts);
|
||||
await migrateStorage(source, client, counts);
|
||||
await migrateYearbook(source, client, counts, archive);
|
||||
const writeClient = apply ? client : null;
|
||||
const cursorSpecs = [
|
||||
['hall', 'id'],
|
||||
['ng_old_generals', 'id'],
|
||||
['ng_old_nations', 'id'],
|
||||
['emperior', 'no'],
|
||||
['inheritance_result', 'id'],
|
||||
['user_record', 'id'],
|
||||
['ng_history', 'no'],
|
||||
] as const;
|
||||
const cursors = new Map<string, AppendCursor>();
|
||||
for (const [sourceTable, idColumn] of cursorSpecs) {
|
||||
const endAtId = (await sourceMaxId(source, sourceTable, idColumn)) ?? -1n;
|
||||
const checkpoint =
|
||||
execution.mode === 'incremental' && client
|
||||
? await loadCheckpoint(
|
||||
client,
|
||||
{
|
||||
tableSql: '"legacy_archive"."import_checkpoint"',
|
||||
scope: { columnSql: '"source_profile"', value: profile },
|
||||
},
|
||||
execution.source.key,
|
||||
sourceTable
|
||||
)
|
||||
: null;
|
||||
const afterId =
|
||||
execution.mode === 'incremental'
|
||||
? requireIncrementalCheckpoint(checkpoint, execution.source, sourceTable)
|
||||
: -1n;
|
||||
if (endAtId < afterId) {
|
||||
throw new Error(`Legacy source ${sourceTable} maximum ID regressed; run a reviewed full migration`);
|
||||
}
|
||||
cursors.set(sourceTable, { afterId, endAtId });
|
||||
}
|
||||
const cursor = (table: string): AppendCursor => {
|
||||
const value = cursors.get(table);
|
||||
if (!value) throw new Error(`Missing migration cursor for ${table}`);
|
||||
return value;
|
||||
};
|
||||
const maxGameId = (await sourceMaxId(source, 'ng_games', 'id')) ?? -1n;
|
||||
await migrateGames(source, writeClient, counts, archive, progress, { afterId: -1n, endAtId: maxGameId });
|
||||
await migrateHall(source, writeClient, counts, archive, progress, cursor('hall'));
|
||||
await migrateOldGenerals(source, writeClient, counts, archive, progress, cursor('ng_old_generals'));
|
||||
await migrateOldNations(source, writeClient, counts, archive, progress, cursor('ng_old_nations'));
|
||||
await migrateEmperors(source, writeClient, counts, archive, progress, cursor('emperior'));
|
||||
await migrateInheritanceResults(source, writeClient, counts, progress, cursor('inheritance_result'));
|
||||
await migrateUserRecords(source, writeClient, counts, progress, cursor('user_record'));
|
||||
await migrateStorage(source, writeClient, counts);
|
||||
progress.storage = {
|
||||
strategy: 'rescan',
|
||||
startAfterId: null,
|
||||
endAtId: null,
|
||||
processed: counts.storage_inspected ?? 0,
|
||||
};
|
||||
await migrateYearbook(source, writeClient, counts, archive, progress, cursor('ng_history'));
|
||||
if (apply && client && archive.importRunId !== '0') {
|
||||
for (const [sourceTable] of cursorSpecs) {
|
||||
await saveCheckpoint(
|
||||
client,
|
||||
{
|
||||
tableSql: '"legacy_archive"."import_checkpoint"',
|
||||
scope: { columnSql: '"source_profile"', value: profile },
|
||||
},
|
||||
execution.source,
|
||||
sourceTable,
|
||||
cursor(sourceTable).endAtId,
|
||||
archive.importRunId
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
if (client) {
|
||||
await withMigrationLock(client, `sammo-legacy-archive-v2:${profile}`, async () => {
|
||||
if (client && apply) {
|
||||
await withMigrationLock(client, `sammo-legacy-archive-v3:${profile}:${execution.source.key}`, async () => {
|
||||
const created = await client.query<{ id: string }>(
|
||||
`INSERT INTO "legacy_archive"."import_run" ("source_profile", "status")
|
||||
VALUES ($1, 'RUNNING') RETURNING "id"`,
|
||||
[profile]
|
||||
`INSERT INTO "legacy_archive"."import_run"
|
||||
("source_profile", "source_key", "source_fingerprint", "mode", "status")
|
||||
VALUES ($1, $2, $3, $4, 'RUNNING') RETURNING "id"`,
|
||||
[profile, execution.source.key, execution.source.fingerprint, execution.mode]
|
||||
);
|
||||
importRunId = created.rows[0]?.id ?? null;
|
||||
if (!importRunId) throw new Error('Failed to create legacy archive import run');
|
||||
@@ -523,10 +649,11 @@ export const migrateGame = async (
|
||||
await run(archive);
|
||||
await client.query(
|
||||
`UPDATE "legacy_archive"."import_run"
|
||||
SET "status" = 'COMPLETED', "finished_at" = CURRENT_TIMESTAMP,
|
||||
"counts" = $2::jsonb, "source_format_summary" = $3::jsonb
|
||||
WHERE "id" = $1`,
|
||||
[importRunId, JSON.stringify(counts), JSON.stringify(sourceFormats)]
|
||||
SET "status" = 'COMPLETED', "finished_at" = CURRENT_TIMESTAMP,
|
||||
"counts" = $2::jsonb, "source_format_summary" = $3::jsonb,
|
||||
"progress" = $4::jsonb
|
||||
WHERE "id" = $1`,
|
||||
[importRunId, JSON.stringify(counts), JSON.stringify(sourceFormats), JSON.stringify(progress)]
|
||||
);
|
||||
await client.query('COMMIT');
|
||||
} catch (error) {
|
||||
@@ -535,10 +662,17 @@ export const migrateGame = async (
|
||||
error instanceof Error ? error.message.slice(0, 2000) : String(error).slice(0, 2000);
|
||||
await client.query(
|
||||
`UPDATE "legacy_archive"."import_run"
|
||||
SET "status" = 'FAILED', "finished_at" = CURRENT_TIMESTAMP,
|
||||
"counts" = $2::jsonb, "source_format_summary" = $3::jsonb, "error" = $4
|
||||
WHERE "id" = $1`,
|
||||
[importRunId, JSON.stringify(counts), JSON.stringify(sourceFormats), message]
|
||||
SET "status" = 'FAILED', "finished_at" = CURRENT_TIMESTAMP,
|
||||
"counts" = $2::jsonb, "source_format_summary" = $3::jsonb,
|
||||
"progress" = $4::jsonb, "error" = $5
|
||||
WHERE "id" = $1`,
|
||||
[
|
||||
importRunId,
|
||||
JSON.stringify(counts),
|
||||
JSON.stringify(sourceFormats),
|
||||
JSON.stringify(progress),
|
||||
message,
|
||||
]
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
@@ -549,5 +683,15 @@ export const migrateGame = async (
|
||||
} finally {
|
||||
client?.release();
|
||||
}
|
||||
return { command: 'game', apply, counts, excluded, importRunId, sourceFormatSummary: sourceFormats };
|
||||
return {
|
||||
command: 'game',
|
||||
apply,
|
||||
counts,
|
||||
excluded,
|
||||
importRunId,
|
||||
sourceFormatSummary: sourceFormats,
|
||||
mode: execution.mode,
|
||||
sourceKey: execution.source.key,
|
||||
progress,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
paginateSource,
|
||||
jsonParameter,
|
||||
querySource,
|
||||
sourceMaxId,
|
||||
toBigInt,
|
||||
toDate,
|
||||
toNullableDate,
|
||||
@@ -17,6 +18,15 @@ import {
|
||||
type SourceRow,
|
||||
type TargetRow,
|
||||
} from './db.js';
|
||||
import {
|
||||
defaultExecutionOptions,
|
||||
loadCheckpoint,
|
||||
requireIncrementalCheckpoint,
|
||||
saveCheckpoint,
|
||||
validateSourceIdentity,
|
||||
type MigrationExecutionOptions,
|
||||
type MigrationProgress,
|
||||
} from './incremental.js';
|
||||
import { mapLegacyRoles, mapLegacySanctions, parseJson, type JsonValue } from './transform.js';
|
||||
|
||||
export interface MigrationSummary {
|
||||
@@ -26,6 +36,9 @@ export interface MigrationSummary {
|
||||
excluded: Record<string, string>;
|
||||
importRunId?: string | null;
|
||||
sourceFormatSummary?: Record<string, number>;
|
||||
mode?: 'full' | 'incremental';
|
||||
sourceKey?: string;
|
||||
progress?: MigrationProgress;
|
||||
}
|
||||
|
||||
const batchSize = 500;
|
||||
@@ -183,9 +196,10 @@ const processMembers = async (
|
||||
const processMemberLogs = async (
|
||||
source: MariaPool,
|
||||
target: PoolClient | null,
|
||||
counts: Record<string, number>
|
||||
counts: Record<string, number>,
|
||||
afterId = -1n
|
||||
): Promise<void> => {
|
||||
for await (const rows of paginateSource(source, 'member_log', 'id', batchSize)) {
|
||||
for await (const rows of paginateSource(source, 'member_log', 'id', batchSize, afterId)) {
|
||||
const mapped = rows.map<TargetRow>((row) => {
|
||||
const id = toBigInt(row.id, 'member_log.id');
|
||||
const memberNo = toNumber(row.member_no, `member_log.${id}.member_no`);
|
||||
@@ -270,38 +284,126 @@ export const migrateGateway = async (
|
||||
source: MariaPool,
|
||||
targetPool: PgPool | null,
|
||||
apply: boolean,
|
||||
migratedAt: Date
|
||||
migratedAt: Date,
|
||||
execution: MigrationExecutionOptions = defaultExecutionOptions('legacy-root')
|
||||
): Promise<MigrationSummary> => {
|
||||
validateSourceIdentity(execution.source);
|
||||
if (execution.mode === 'incremental' && !targetPool) {
|
||||
throw new Error('Incremental gateway migration requires the target database to read checkpoints');
|
||||
}
|
||||
const counts: Record<string, number> = {};
|
||||
const progress: MigrationProgress = {};
|
||||
const excluded = {
|
||||
login_token:
|
||||
'Legacy bearer tokens, IP addresses, and expired sessions are not valid in the Redis session model.',
|
||||
};
|
||||
const client = targetPool ? await targetPool.connect() : null;
|
||||
let importRunId: string | null = null;
|
||||
try {
|
||||
const run = async (): Promise<void> => {
|
||||
const run = async (runId: string | null): Promise<void> => {
|
||||
await processMembers(source, client, apply, migratedAt, counts);
|
||||
await processMemberLogs(source, apply ? client : null, counts);
|
||||
progress.member = {
|
||||
strategy: 'rescan',
|
||||
startAfterId: null,
|
||||
endAtId: null,
|
||||
processed: counts.member ?? 0,
|
||||
};
|
||||
|
||||
const maxMemberLogId = (await sourceMaxId(source, 'member_log', 'id')) ?? -1n;
|
||||
const checkpoint =
|
||||
execution.mode === 'incremental' && client
|
||||
? await loadCheckpoint(
|
||||
client,
|
||||
{ tableSql: '"legacy_import_checkpoint"' },
|
||||
execution.source.key,
|
||||
'member_log'
|
||||
)
|
||||
: null;
|
||||
const memberLogAfter =
|
||||
execution.mode === 'incremental'
|
||||
? requireIncrementalCheckpoint(checkpoint, execution.source, 'member_log')
|
||||
: -1n;
|
||||
if (maxMemberLogId < memberLogAfter) {
|
||||
throw new Error('Legacy source member_log maximum ID regressed; run a reviewed full migration');
|
||||
}
|
||||
await processMemberLogs(source, apply ? client : null, counts, memberLogAfter);
|
||||
progress.member_log = {
|
||||
strategy: 'append',
|
||||
startAfterId: memberLogAfter < 0n ? null : memberLogAfter.toString(),
|
||||
endAtId: maxMemberLogId < 0n ? null : maxMemberLogId.toString(),
|
||||
processed: counts.member_log ?? 0,
|
||||
};
|
||||
await processBannedMembers(source, apply ? client : null, counts);
|
||||
await processRootKeyValues(source, apply ? client : null, counts);
|
||||
await processSystem(source, apply ? client : null, counts);
|
||||
for (const table of ['banned_member', 'root_key_value', 'system'] as const) {
|
||||
progress[table] = {
|
||||
strategy: 'rescan',
|
||||
startAfterId: null,
|
||||
endAtId: null,
|
||||
processed: counts[table] ?? 0,
|
||||
};
|
||||
}
|
||||
if (apply && client && runId) {
|
||||
await saveCheckpoint(
|
||||
client,
|
||||
{ tableSql: '"legacy_import_checkpoint"' },
|
||||
execution.source,
|
||||
'member_log',
|
||||
maxMemberLogId,
|
||||
runId
|
||||
);
|
||||
}
|
||||
};
|
||||
if (client && apply) {
|
||||
await withMigrationLock(client, 'sammo-legacy-gateway-v1', async () => {
|
||||
await withMigrationLock(client, `sammo-legacy-gateway-v2:${execution.source.key}`, async () => {
|
||||
const created = await client.query<{ id: string }>(
|
||||
`INSERT INTO "legacy_import_run"
|
||||
("source_key", "source_fingerprint", "mode", "status")
|
||||
VALUES ($1, $2, $3, 'RUNNING') RETURNING "id"`,
|
||||
[execution.source.key, execution.source.fingerprint, execution.mode]
|
||||
);
|
||||
importRunId = created.rows[0]?.id ?? null;
|
||||
if (!importRunId) throw new Error('Failed to create legacy gateway import run');
|
||||
await client.query('BEGIN');
|
||||
try {
|
||||
await run();
|
||||
await run(importRunId);
|
||||
await client.query(
|
||||
`UPDATE "legacy_import_run"
|
||||
SET "status" = 'COMPLETED', "finished_at" = CURRENT_TIMESTAMP,
|
||||
"counts" = $2::jsonb, "progress" = $3::jsonb
|
||||
WHERE "id" = $1`,
|
||||
[importRunId, JSON.stringify(counts), JSON.stringify(progress)]
|
||||
);
|
||||
await client.query('COMMIT');
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
const message =
|
||||
error instanceof Error ? error.message.slice(0, 2000) : String(error).slice(0, 2000);
|
||||
await client.query(
|
||||
`UPDATE "legacy_import_run"
|
||||
SET "status" = 'FAILED', "finished_at" = CURRENT_TIMESTAMP,
|
||||
"counts" = $2::jsonb, "progress" = $3::jsonb, "error" = $4
|
||||
WHERE "id" = $1`,
|
||||
[importRunId, JSON.stringify(counts), JSON.stringify(progress), message]
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
} else {
|
||||
await run();
|
||||
await run(null);
|
||||
}
|
||||
} finally {
|
||||
client?.release();
|
||||
}
|
||||
return { command: 'gateway', apply, counts, excluded };
|
||||
return {
|
||||
command: 'gateway',
|
||||
apply,
|
||||
counts,
|
||||
excluded,
|
||||
importRunId,
|
||||
mode: execution.mode,
|
||||
sourceKey: execution.source.key,
|
||||
progress,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
import type { PoolClient } from 'pg';
|
||||
|
||||
export type MigrationMode = 'full' | 'incremental';
|
||||
|
||||
export interface MigrationSourceIdentity {
|
||||
key: string;
|
||||
fingerprint: string;
|
||||
}
|
||||
|
||||
export interface MigrationExecutionOptions {
|
||||
mode: MigrationMode;
|
||||
source: MigrationSourceIdentity;
|
||||
}
|
||||
|
||||
export interface MigrationTableProgress {
|
||||
strategy: 'append' | 'rescan';
|
||||
startAfterId: string | null;
|
||||
endAtId: string | null;
|
||||
processed: number;
|
||||
}
|
||||
|
||||
export type MigrationProgress = Record<string, MigrationTableProgress>;
|
||||
|
||||
export interface StoredCheckpoint {
|
||||
sourceFingerprint: string;
|
||||
lastLegacyId: bigint;
|
||||
}
|
||||
|
||||
export interface CheckpointStore {
|
||||
tableSql: '"legacy_import_checkpoint"' | '"legacy_archive"."import_checkpoint"';
|
||||
scope?: { columnSql: '"source_profile"'; value: string };
|
||||
}
|
||||
|
||||
const SOURCE_KEY = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
||||
|
||||
export const validateSourceIdentity = (source: MigrationSourceIdentity): MigrationSourceIdentity => {
|
||||
if (!SOURCE_KEY.test(source.key)) {
|
||||
throw new Error('Legacy migration source key must use 1-128 safe characters');
|
||||
}
|
||||
if (!/^[a-f0-9]{64}$/u.test(source.fingerprint)) {
|
||||
throw new Error('Legacy migration source fingerprint must be a SHA-256 hex value');
|
||||
}
|
||||
return source;
|
||||
};
|
||||
|
||||
export const fingerprintMariaConnection = (connectionString: string): string => {
|
||||
const url = new URL(connectionString);
|
||||
if (url.protocol !== 'mariadb:' && url.protocol !== 'mysql:') {
|
||||
throw new Error('Legacy source URL must use the mariadb or mysql protocol');
|
||||
}
|
||||
const query = [...url.searchParams.entries()]
|
||||
.filter(([key]) => !/pass(word)?|secret|token/iu.test(key))
|
||||
.sort(([leftKey, leftValue], [rightKey, rightValue]) =>
|
||||
leftKey === rightKey ? leftValue.localeCompare(rightValue) : leftKey.localeCompare(rightKey)
|
||||
);
|
||||
const identity = {
|
||||
protocol: url.protocol,
|
||||
host: url.hostname.toLowerCase(),
|
||||
port: url.port || '3306',
|
||||
database: decodeURIComponent(url.pathname.replace(/^\//u, '')),
|
||||
user: decodeURIComponent(url.username),
|
||||
query,
|
||||
};
|
||||
return createHash('sha256').update(JSON.stringify(identity)).digest('hex');
|
||||
};
|
||||
|
||||
export const loadCheckpoint = async (
|
||||
client: PoolClient,
|
||||
store: CheckpointStore,
|
||||
sourceKey: string,
|
||||
sourceTable: string
|
||||
): Promise<StoredCheckpoint | null> => {
|
||||
const scopePredicate = store.scope ? ` AND ${store.scope.columnSql} = $3` : '';
|
||||
const parameters = store.scope ? [sourceKey, sourceTable, store.scope.value] : [sourceKey, sourceTable];
|
||||
const result = await client.query<{ source_fingerprint: string; last_legacy_id: string }>(
|
||||
`SELECT "source_fingerprint", "last_legacy_id"
|
||||
FROM ${store.tableSql}
|
||||
WHERE "source_key" = $1 AND "source_table" = $2${scopePredicate}
|
||||
FOR UPDATE`,
|
||||
parameters
|
||||
);
|
||||
const row = result.rows[0];
|
||||
return row ? { sourceFingerprint: row.source_fingerprint, lastLegacyId: BigInt(row.last_legacy_id) } : null;
|
||||
};
|
||||
|
||||
export const requireIncrementalCheckpoint = (
|
||||
checkpoint: StoredCheckpoint | null,
|
||||
source: MigrationSourceIdentity,
|
||||
sourceTable: string
|
||||
): bigint => {
|
||||
if (!checkpoint) {
|
||||
throw new Error(`Incremental migration requires a completed full checkpoint for ${sourceTable}`);
|
||||
}
|
||||
if (checkpoint.sourceFingerprint !== source.fingerprint) {
|
||||
throw new Error(`Legacy source fingerprint changed for ${sourceTable}; run a reviewed full migration`);
|
||||
}
|
||||
return checkpoint.lastLegacyId;
|
||||
};
|
||||
|
||||
export const saveCheckpoint = async (
|
||||
client: PoolClient,
|
||||
store: CheckpointStore,
|
||||
source: MigrationSourceIdentity,
|
||||
sourceTable: string,
|
||||
lastLegacyId: bigint,
|
||||
importRunId: string
|
||||
): Promise<void> => {
|
||||
const scopeColumn = store.scope ? `${store.scope.columnSql}, ` : '';
|
||||
const scopeValue = store.scope ? '$1, ' : '';
|
||||
const parameterOffset = store.scope ? 1 : 0;
|
||||
const parameters: unknown[] = store.scope ? [store.scope.value] : [];
|
||||
parameters.push(source.key, source.fingerprint, sourceTable, lastLegacyId.toString(), importRunId);
|
||||
const conflictColumns = store.scope
|
||||
? `${store.scope.columnSql}, "source_key", "source_table"`
|
||||
: '"source_key", "source_table"';
|
||||
await client.query(
|
||||
`INSERT INTO ${store.tableSql}
|
||||
(${scopeColumn}"source_key", "source_fingerprint", "source_table", "last_legacy_id", "import_run_id", "updated_at")
|
||||
VALUES (${scopeValue}$${parameterOffset + 1}, $${parameterOffset + 2}, $${parameterOffset + 3}, $${parameterOffset + 4}, $${parameterOffset + 5}, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (${conflictColumns}) DO UPDATE SET
|
||||
"source_fingerprint" = EXCLUDED."source_fingerprint",
|
||||
"last_legacy_id" = EXCLUDED."last_legacy_id",
|
||||
"import_run_id" = EXCLUDED."import_run_id",
|
||||
"updated_at" = CURRENT_TIMESTAMP`,
|
||||
parameters
|
||||
);
|
||||
};
|
||||
|
||||
export const defaultExecutionOptions = (
|
||||
sourceKey: string,
|
||||
connectionString = `mariadb://legacy@localhost/${sourceKey}`
|
||||
): MigrationExecutionOptions => ({
|
||||
mode: 'full',
|
||||
source: { key: sourceKey, fingerprint: fingerprintMariaConnection(connectionString) },
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { createMariaPool, createPostgresPool, querySource } from './db.js';
|
||||
import { migrateGame } from './game.js';
|
||||
import { migrateGateway, type MigrationSummary } from './gateway.js';
|
||||
import type { MigrationMode } from './incremental.js';
|
||||
import type { ResolvedMigrationPlan, ResolvedMigrationStage } from './config.js';
|
||||
|
||||
export interface PlanRunSummary {
|
||||
command: 'run-plan';
|
||||
sourceSet: string;
|
||||
mode: MigrationMode;
|
||||
apply: boolean;
|
||||
stages: Array<{ name: string; status: 'COMPLETED'; summary: MigrationSummary }>;
|
||||
}
|
||||
|
||||
const preflightStage = async (stage: ResolvedMigrationStage): Promise<void> => {
|
||||
const source = createMariaPool(stage.sourceUrl);
|
||||
const target = createPostgresPool(stage.targetUrl);
|
||||
try {
|
||||
const sourceDatabase = await querySource(source, 'SELECT DATABASE() AS database_name');
|
||||
if (typeof sourceDatabase[0]?.database_name !== 'string') {
|
||||
throw new Error(`Source preflight did not select a database for ${stage.name}`);
|
||||
}
|
||||
const requiredSourceTables =
|
||||
stage.kind === 'gateway'
|
||||
? ['member', 'member_log', 'banned_member', 'storage', 'system']
|
||||
: [
|
||||
'ng_games',
|
||||
'hall',
|
||||
'ng_old_generals',
|
||||
'ng_old_nations',
|
||||
'emperior',
|
||||
'inheritance_result',
|
||||
'user_record',
|
||||
'storage',
|
||||
'ng_history',
|
||||
];
|
||||
const sourceTables = await querySource(
|
||||
source,
|
||||
`SELECT table_name AS source_table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = DATABASE() AND table_name IN (${requiredSourceTables.map(() => '?').join(', ')})`,
|
||||
requiredSourceTables
|
||||
);
|
||||
const availableSourceTables = new Set(sourceTables.map((row) => String(row.source_table_name)));
|
||||
const missingSourceTables = requiredSourceTables.filter((table) => !availableSourceTables.has(table));
|
||||
if (missingSourceTables.length) {
|
||||
throw new Error(`Source ${stage.name} is missing required tables: ${missingSourceTables.join(', ')}`);
|
||||
}
|
||||
await target.query('SELECT 1');
|
||||
const checkpointTable =
|
||||
stage.kind === 'gateway' ? 'public.legacy_import_checkpoint' : 'legacy_archive.import_checkpoint';
|
||||
const migrationReady = await target.query<{ table_name: string | null }>(
|
||||
'SELECT to_regclass($1) AS table_name',
|
||||
[checkpointTable]
|
||||
);
|
||||
if (!migrationReady.rows[0]?.table_name) {
|
||||
throw new Error(`Target migrations are not current for ${stage.name}; missing ${checkpointTable}`);
|
||||
}
|
||||
const targetDataTable = stage.kind === 'gateway' ? 'public.app_user' : 'inheritance_result';
|
||||
const targetReady = await target.query<{ table_name: string | null }>('SELECT to_regclass($1) AS table_name', [
|
||||
targetDataTable,
|
||||
]);
|
||||
if (!targetReady.rows[0]?.table_name) {
|
||||
throw new Error(`Target migrations are not current for ${stage.name}; missing ${targetDataTable}`);
|
||||
}
|
||||
} finally {
|
||||
await source.end();
|
||||
await target.end();
|
||||
}
|
||||
};
|
||||
|
||||
export const checkMigrationPlan = async (plan: ResolvedMigrationPlan): Promise<Record<string, unknown>> => {
|
||||
for (const stage of plan.stages) await preflightStage(stage);
|
||||
return {
|
||||
command: 'check-plan',
|
||||
sourceSet: plan.sourceSet,
|
||||
stages: plan.stages.map((stage) => ({ name: stage.name, kind: stage.kind, status: 'READY' })),
|
||||
};
|
||||
};
|
||||
|
||||
export const runMigrationPlan = async (
|
||||
plan: ResolvedMigrationPlan,
|
||||
mode: MigrationMode,
|
||||
apply: boolean,
|
||||
migratedAt = new Date()
|
||||
): Promise<PlanRunSummary> => {
|
||||
await checkMigrationPlan(plan);
|
||||
const stages: PlanRunSummary['stages'] = [];
|
||||
for (const stage of plan.stages) {
|
||||
const source = createMariaPool(stage.sourceUrl);
|
||||
const target = createPostgresPool(stage.targetUrl);
|
||||
try {
|
||||
const execution = { mode, source: stage.sourceIdentity } as const;
|
||||
const summary =
|
||||
stage.kind === 'gateway'
|
||||
? await migrateGateway(source, target, apply, migratedAt, execution)
|
||||
: await migrateGame(source, target, apply, stage.profile!, execution);
|
||||
stages.push({ name: stage.name, status: 'COMPLETED', summary });
|
||||
} finally {
|
||||
await source.end();
|
||||
await target.end();
|
||||
}
|
||||
}
|
||||
return { command: 'run-plan', sourceSet: plan.sourceSet, mode, apply, stages };
|
||||
};
|
||||
Reference in New Issue
Block a user