feat: 레거시 DB 순차 증분 이관 CLI 추가

2026-08-18 12:35:24 +00:00
parent e910a635e6
commit da0e32f422
21 changed files with 1375 additions and 104 deletions
@@ -0,0 +1,60 @@
+import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { loadMigrationPlan } from '../src/config.js';
+
+const workDirectories: string[] = [];
+
+afterEach(async () => {
+    delete process.env.TEST_GATEWAY_DATABASE_URL;
+    await Promise.all(workDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })));
+});
+
+const writeFixture = async (mode = 0o600): Promise<string> => {
+    const directory = await mkdtemp(path.join(os.tmpdir(), 'sammo-legacy-plan-'));
+    workDirectories.push(directory);
+    await writeFile(path.join(directory, 'mysql-password'), 'secret-value\n', { mode: 0o600 });
+    const configPath = path.join(directory, 'migration-plan.json');
+    await writeFile(
+        configPath,
+        JSON.stringify({
+            version: 1,
+            sourceSet: 'fixture-cutover',
+            gateway: {
+                source: {
+                    host: '127.0.0.1',
+                    database: 'root_dump',
+                    user: 'migration_reader',
+                    passwordFile: './mysql-password',
+                },
+                targetUrlEnv: 'TEST_GATEWAY_DATABASE_URL',
+            },
+        }),
+        { mode }
+    );
+    await chmod(configPath, mode);
+    return configPath;
+};
+
+describe('legacy migration plan config', () => {
+    it('resolves a mode-0600 structured source without placing its password in arguments', async () => {
+        process.env.TEST_GATEWAY_DATABASE_URL = 'postgresql://target@127.0.0.1/gateway';
+        const plan = await loadMigrationPlan(await writeFixture());
+        const source = new URL(plan.stages[0]!.sourceUrl);
+
+        expect(plan.sourceSet).toBe('fixture-cutover');
+        expect(plan.stages.map((stage) => stage.name)).toEqual(['gateway']);
+        expect(source.hostname).toBe('127.0.0.1');
+        expect(source.username).toBe('migration_reader');
+        expect(source.password).toBe('secret-value');
+        expect(plan.stages[0]!.sourceIdentity.key).toBe('fixture-cutover:gateway');
+    });
+
+    it('rejects a config readable by group or other users', async () => {
+        process.env.TEST_GATEWAY_DATABASE_URL = 'postgresql://target@127.0.0.1/gateway';
+        await expect(loadMigrationPlan(await writeFixture(0o644))).rejects.toThrow('expected mode 0600');
+    });
+});
@@ -0,0 +1,38 @@
+USE root_legacy;
+
+UPDATE `member` SET `REG_NUM` = 1 WHERE `NO` = 1;
+UPDATE `system` SET `NOTICE` = 'incremental notice', `MDF_DATE` = '2020-02-01 00:00:00' WHERE `NO` = 1;
+UPDATE `storage` SET `value` = '{"version":2}' WHERE `id` = 1;
+INSERT INTO `member_log` (`id`, `member_no`, `date`, `action_type`, `action`)
+VALUES (2, 1, '2020-02-01 00:00:00', 'logout', NULL);
+
+USE che_legacy;
+
+UPDATE `ng_games` SET `winner_nation` = 1 WHERE `id` = 1;
+UPDATE `storage` SET `value` = '[20,null]' WHERE `id` = 1;
+
+INSERT INTO `hall`
+    (`id`, `server_id`, `season`, `scenario`, `general_no`, `type`, `value`, `owner`, `aux`)
+VALUES (2, 'che_fixture_002', 2, 2, 11, 'war', 200, 1, '{}');
+
+INSERT INTO `ng_old_generals`
+    (`id`, `server_id`, `general_no`, `owner`, `name`, `last_yearmonth`, `turntime`, `data`)
+VALUES
+    (2, 'che_fixture_002', 11, 1, 'Incremental General', 22112, '2020-02-01 00:00:00.000000',
+     '{"leader":81,"power":71,"intel":61,"history":"second<br>"}');
+
+INSERT INTO `ng_old_nations` (`id`, `server_id`, `nation`, `data`, `date`)
+VALUES (2, 'che_fixture_002', 2, '{}', '2020-02-01 00:00:00');
+
+INSERT INTO `emperior` (`no`, `server_id`, `name`, `history`, `aux`)
+VALUES (2, 'che_fixture_002', 'Incremental Emperor', '[]', '{}');
+
+INSERT INTO `inheritance_result` (`id`, `server_id`, `owner`, `general_id`, `year`, `month`, `value`)
+VALUES (2, 'che_fixture_002', 1, 11, 221, 12, '{}');
+
+INSERT INTO `user_record` (`id`, `user_id`, `server_id`, `log_type`, `year`, `month`, `date`, `text`)
+VALUES (2, 1, 'che_fixture_002', 'history', 221, 12, '2020-02-01 00:00:00', 'incremental history');
+
+INSERT INTO `ng_history`
+    (`no`, `server_id`, `year`, `month`, `map`, `global_history`, `global_action`, `nations`)
+VALUES (2, 'che_fixture_002', 221, 12, '{}', '[]', '[]', '[]');
@@ -0,0 +1,56 @@
+USE root_legacy;
+
+INSERT INTO `system` (`NO`, `REG`, `LOGIN`, `NOTICE`, `CRT_DATE`, `MDF_DATE`)
+VALUES (1, 'Y', 'Y', 'initial notice', '2020-01-01 00:00:00', '2020-01-01 00:00:00');
+
+INSERT INTO `member`
+    (`NO`, `oauth_id`, `ID`, `EMAIL`, `oauth_type`, `oauth_info`, `token_valid_until`, `PW`, `salt`,
+     `third_use`, `NAME`, `PICTURE`, `IMGSVR`, `acl`, `penalty`, `GRADE`, `REG_NUM`, `REG_DATE`,
+     `BLOCK_NUM`, `BLOCK_DATE`, `delete_after`)
+VALUES
+    (1, NULL, 'fixture-user', 'fixture@example.test', 'NONE', '{}', NULL,
+     REPEAT('a', 128), 'fixture-salt-001', 0, 'Fixture User', 'default.jpg', 0, '{}', '{}', 1, 0,
+     '2020-01-01 00:00:00', 0, NULL, NULL);
+
+INSERT INTO `member_log` (`id`, `member_no`, `date`, `action_type`, `action`)
+VALUES (1, 1, '2020-01-01 00:00:00', 'login', NULL);
+
+INSERT INTO `storage` (`id`, `namespace`, `key`, `value`)
+VALUES (1, 'fixture', 'mutable', '{"version":1}');
+
+USE che_legacy;
+
+INSERT INTO `ng_games`
+    (`id`, `server_id`, `date`, `winner_nation`, `map`, `season`, `scenario`, `scenario_name`, `env`)
+VALUES
+    (1, 'che_fixture_001', '2020-01-01 00:00:00', NULL, 'che', 1, 2, 'fixture',
+     '{"opentime":"2020-01-01 00:00:00"}');
+
+INSERT INTO `hall`
+    (`id`, `server_id`, `season`, `scenario`, `general_no`, `type`, `value`, `owner`, `aux`)
+VALUES (1, 'che_fixture_001', 1, 2, 10, 'war', 100, 1, '{}');
+
+INSERT INTO `ng_old_generals`
+    (`id`, `server_id`, `general_no`, `owner`, `name`, `last_yearmonth`, `turntime`, `data`)
+VALUES
+    (1, 'che_fixture_001', 10, 1, 'Fixture General', 22012, '2020-01-01 00:00:00.000000',
+     '{"leader":80,"power":70,"intel":60,"history":"first<br>"}');
+
+INSERT INTO `ng_old_nations` (`id`, `server_id`, `nation`, `data`, `date`)
+VALUES (1, 'che_fixture_001', 1, '{}', '2020-01-01 00:00:00');
+
+INSERT INTO `emperior` (`no`, `server_id`, `name`, `history`, `aux`)
+VALUES (1, 'che_fixture_001', 'Fixture Emperor', '[]', '{}');
+
+INSERT INTO `inheritance_result` (`id`, `server_id`, `owner`, `general_id`, `year`, `month`, `value`)
+VALUES (1, 'che_fixture_001', 1, 10, 220, 12, '{}');
+
+INSERT INTO `user_record` (`id`, `user_id`, `server_id`, `log_type`, `year`, `month`, `date`, `text`)
+VALUES (1, 1, 'che_fixture_001', 'history', 220, 12, '2020-01-01 00:00:00', 'fixture history');
+
+INSERT INTO `storage` (`id`, `namespace`, `key`, `value`)
+VALUES (1, 'inheritance_1', 'point', '[10,null]');
+
+INSERT INTO `ng_history`
+    (`no`, `server_id`, `year`, `month`, `map`, `global_history`, `global_action`, `nations`)
+VALUES (1, 'che_fixture_001', 220, 12, '{}', '[]', '[]', '[]');
@@ -35,16 +35,30 @@ const sourceRows = {
 const sourcePool = (): MariaPool => {
    const seen = new Set<string>();
    return {
-        query: vi.fn(async (sql: string) => {
+        query: vi.fn(async (sql: string, values: readonly unknown[] = []) => {
            const table = /FROM `([a-z_]+)`/u.exec(sql)?.[1] ?? '';
+            if (sql.includes('SELECT MAX(')) {
+                const rows = (sourceRows[table as keyof typeof sourceRows] ?? []) as Array<Record<string, unknown>>;
+                const idColumn = /MAX\(`([a-z_]+)`\)/u.exec(sql)?.[1] ?? 'id';
+                return [{ max_id: rows.length ? rows.at(-1)?.[idColumn] : null }];
+            }
            if (seen.has(table)) return [];
            seen.add(table);
-            return sourceRows[table as keyof typeof sourceRows] ?? [];
+            const afterId = BigInt(String(values[0] ?? -1));
+            return ((sourceRows[table as keyof typeof sourceRows] ?? []) as Array<Record<string, unknown>>).filter(
+                (row) => {
+                    const idColumn = /WHERE `([a-z_]+)` >/u.exec(sql)?.[1] ?? 'id';
+                    return BigInt(String(row[idColumn])) > afterId;
+                }
+            );
        }),
    } as unknown as MariaPool;
 };

-const targetPool = (failPattern?: string) => {
+const targetPool = (
+    failPattern?: string,
+    checkpoints?: Record<string, { fingerprint: string; lastLegacyId: string }>
+) => {
    const queries: Array<{ sql: string; values: readonly unknown[] }> = [];
    const query = vi.fn(async (sql: string, values: readonly unknown[] = []) => {
        queries.push({ sql, values });
@@ -55,6 +69,20 @@ const targetPool = (failPattern?: string) => {
        if (sql.includes('INSERT INTO "legacy_archive"."import_run"')) {
            return { rows: [{ id: '77' }], rowCount: 1 } as QueryResult<{ id: string }>;
        }
+        if (sql.includes('FROM "legacy_archive"."import_checkpoint"')) {
+            const checkpoint = checkpoints?.[String(values[1])];
+            return {
+                rows: checkpoint
+                    ? [
+                          {
+                              source_fingerprint: checkpoint.fingerprint,
+                              last_legacy_id: checkpoint.lastLegacyId,
+                          },
+                      ]
+                    : [],
+                rowCount: checkpoint ? 1 : 0,
+            } as unknown as QueryResult;
+        }
        return { rows: [], rowCount: 0 } as unknown as QueryResult;
    });
    const client = { query, release: vi.fn() } as unknown as PoolClient;
@@ -143,4 +171,39 @@ describe('legacy archive game migration', () => {
            'Unsupported legacy archive profile'
        );
    });
+
+    it('uses checkpoints for append-only tables while rescanning mutable game history', async () => {
+        const fingerprint = 'a'.repeat(64);
+        const checkpoints = Object.fromEntries(
+            ['hall', 'ng_old_nations', 'emperior', 'inheritance_result', 'user_record', 'ng_history'].map((table) => [
+                table,
+                { fingerprint, lastLegacyId: '-1' },
+            ])
+        );
+        checkpoints.ng_old_generals = { fingerprint, lastLegacyId: '1' };
+        const target = targetPool(undefined, checkpoints);
+
+        const summary = await migrateGame(sourcePool(), target.pool, false, 'che', {
+            mode: 'incremental',
+            source: { key: 'fixture:che', fingerprint },
+        });
+
+        expect(summary.counts).toMatchObject({ ng_games: 1, ng_old_generals: 1 });
+        expect(summary.progress).toMatchObject({
+            ng_games: { strategy: 'rescan', startAfterId: null, processed: 1 },
+            ng_old_generals: { strategy: 'append', startAfterId: '1', endAtId: '2', processed: 1 },
+        });
+        expect(target.queries.some((entry) => entry.sql.includes('INSERT INTO "legacy_archive"."general"'))).toBe(
+            false
+        );
+    });
+
+    it('refuses incremental mode without a completed full checkpoint', async () => {
+        await expect(
+            migrateGame(sourcePool(), targetPool().pool, false, 'che', {
+                mode: 'incremental',
+                source: { key: 'fixture:che', fingerprint: 'a'.repeat(64) },
+            })
+        ).rejects.toThrow('requires a completed full checkpoint');
+    });
 });
@@ -1,8 +1,9 @@
-import type { PoolClient } from 'pg';
+import type { Pool as MariaPool } from 'mariadb';
+import type { Pool as PgPool, PoolClient, QueryResult } from 'pg';
 import { describe, expect, it, vi } from 'vitest';

 import { upsertRows } from '../src/db.js';
-import { mapMember, MEMBER_PRESERVED_COLUMNS, preflightMemberConflicts } from '../src/gateway.js';
+import { mapMember, MEMBER_PRESERVED_COLUMNS, migrateGateway, preflightMemberConflicts } from '../src/gateway.js';

 const memberRow = (overrides: Record<string, unknown> = {}) => ({
    NO: 7,
@@ -110,4 +111,57 @@ describe('legacy gateway member migration', () => {
        );
        expect(String(query.mock.calls[0]?.[0])).toContain('FROM "app_user"');
    });
+
+    it('reads only new immutable member logs during an incremental dry-run', async () => {
+        const fingerprint = 'a'.repeat(64);
+        const seen = new Set<string>();
+        const source = {
+            query: vi.fn(async (sql: string, values: readonly unknown[] = []) => {
+                if (sql.includes('MAX(`id`)') && sql.includes('member_log')) return [{ max_id: 2 }];
+                if (sql.includes('MAX(`date`)')) return [];
+                if (sql.includes('FROM `member`')) return [];
+                if (sql.includes('FROM `member_log`')) {
+                    if (seen.has('member_log')) return [];
+                    seen.add('member_log');
+                    return Number(values[0]) < 2
+                        ? [
+                              {
+                                  id: 2,
+                                  member_no: 7,
+                                  date: new Date('2026-08-18T00:00:00.000Z'),
+                                  action_type: 'login',
+                                  action: null,
+                              },
+                          ]
+                        : [];
+                }
+                return [];
+            }),
+        } as unknown as MariaPool;
+        const query = vi.fn(async (sql: string) => {
+            if (sql.includes('FROM "legacy_import_checkpoint"')) {
+                return {
+                    rows: [{ source_fingerprint: fingerprint, last_legacy_id: '1' }],
+                    rowCount: 1,
+                } as QueryResult;
+            }
+            return { rows: [], rowCount: 0 } as unknown as QueryResult;
+        });
+        const client = { query, release: vi.fn() } as unknown as PoolClient;
+        const target = { connect: vi.fn(async () => client) } as unknown as PgPool;
+
+        const summary = await migrateGateway(source, target, false, new Date('2026-08-18T00:00:00.000Z'), {
+            mode: 'incremental',
+            source: { key: 'fixture:gateway', fingerprint },
+        });
+
+        expect(summary.counts.member_log).toBe(1);
+        expect(summary.progress?.member_log).toMatchObject({
+            strategy: 'append',
+            startAfterId: '1',
+            endAtId: '2',
+            processed: 1,
+        });
+        expect(query.mock.calls.some(([sql]) => String(sql).includes('INSERT INTO "legacy_member_log"'))).toBe(false);
+    });
 });
@@ -0,0 +1,29 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+    fingerprintMariaConnection,
+    requireIncrementalCheckpoint,
+    validateSourceIdentity,
+} from '../src/incremental.js';
+
+describe('legacy incremental source identity', () => {
+    it('excludes passwords while binding host, database, user, and non-secret options', () => {
+        const first = fingerprintMariaConnection('mariadb://reader:first@db.internal:3307/root_dump?ssl=true');
+        const rotated = fingerprintMariaConnection('mariadb://reader:rotated@db.internal:3307/root_dump?ssl=true');
+        const otherDatabase = fingerprintMariaConnection('mariadb://reader:rotated@db.internal:3307/che_dump?ssl=true');
+
+        expect(first).toBe(rotated);
+        expect(first).not.toBe(otherDatabase);
+    });
+
+    it('rejects missing, mismatched, and malformed checkpoint identities', () => {
+        const source = { key: 'cutover:che', fingerprint: 'a'.repeat(64) };
+        expect(() => requireIncrementalCheckpoint(null, source, 'hall')).toThrow('completed full checkpoint');
+        expect(() =>
+            requireIncrementalCheckpoint({ sourceFingerprint: 'b'.repeat(64), lastLegacyId: 1n }, source, 'hall')
+        ).toThrow('fingerprint changed');
+        expect(() => validateSourceIdentity({ key: '../unsafe', fingerprint: 'a'.repeat(64) })).toThrow(
+            'safe characters'
+        );
+    });
+});