perf: share Turbo cache across release worktrees
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
|
||||
export interface BuildCommand {
|
||||
command: string;
|
||||
@@ -25,6 +26,35 @@ export interface BuildRunner {
|
||||
}
|
||||
|
||||
export const MAX_BUILD_OUTPUT_CHARS = 64 * 1024;
|
||||
export const RELEASE_TURBO_CONCURRENCY = 2;
|
||||
|
||||
export const resolveReleaseTurboCacheDir = (cacheAnchorRoot: string, env?: Record<string, string>): string => {
|
||||
const configured = env?.TURBO_CACHE_DIR?.trim();
|
||||
if (!configured) return path.join(path.resolve(cacheAnchorRoot), '.turbo', 'release-cache');
|
||||
return path.isAbsolute(configured) ? configured : path.resolve(cacheAnchorRoot, configured);
|
||||
};
|
||||
|
||||
export const buildTurboReleaseCommand = (
|
||||
workspaceRoot: string,
|
||||
cacheAnchorRoot: string,
|
||||
packageNames: string[],
|
||||
env?: Record<string, string>
|
||||
): BuildCommand => ({
|
||||
command: 'pnpm',
|
||||
args: [
|
||||
'exec',
|
||||
'turbo',
|
||||
'run',
|
||||
'build',
|
||||
...packageNames.map((packageName) => `--filter=${packageName}`),
|
||||
`--cache-dir=${resolveReleaseTurboCacheDir(cacheAnchorRoot, env)}`,
|
||||
`--concurrency=${RELEASE_TURBO_CONCURRENCY}`,
|
||||
'--ui=stream',
|
||||
'--output-logs=new-only',
|
||||
],
|
||||
cwd: workspaceRoot,
|
||||
env,
|
||||
});
|
||||
|
||||
const appendOutputTail = (current: string, chunk: unknown): string =>
|
||||
`${current}${String(chunk)}`.slice(-MAX_BUILD_OUTPUT_CHARS);
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
} from '@sammo-ts/infra';
|
||||
import { isRecord } from '@sammo-ts/common';
|
||||
|
||||
import type { BuildCommand, BuildRunner } from './buildRunner.js';
|
||||
import { buildTurboReleaseCommand, type BuildCommand, type BuildRunner } from './buildRunner.js';
|
||||
import { sanitizeManagedProcessEnv, type ProcessManager } from './processManager.js';
|
||||
import type {
|
||||
GatewayClaimedProfileUpdate,
|
||||
@@ -494,7 +494,8 @@ export const buildProfileFrontendCommands = (
|
||||
export const buildWorkspaceCommands = (
|
||||
workspaceRoot: string,
|
||||
needsInstall: boolean,
|
||||
env?: Record<string, string>
|
||||
env?: Record<string, string>,
|
||||
cacheAnchorRoot: string = workspaceRoot
|
||||
): BuildCommand[] => {
|
||||
const commands: BuildCommand[] = [];
|
||||
if (needsInstall) {
|
||||
@@ -505,23 +506,9 @@ export const buildWorkspaceCommands = (
|
||||
env,
|
||||
});
|
||||
}
|
||||
const buildSteps: Array<[filter: string, script: string]> = [
|
||||
['@sammo-ts/common', 'build'],
|
||||
['@sammo-ts/infra', 'prisma:generate'],
|
||||
['@sammo-ts/infra', 'build'],
|
||||
['@sammo-ts/logic', 'build'],
|
||||
['@sammo-ts/game-api', 'build'],
|
||||
['@sammo-ts/game-engine', 'build'],
|
||||
['@sammo-ts/gateway-api', 'build'],
|
||||
];
|
||||
for (const [filter, script] of buildSteps) {
|
||||
commands.push({
|
||||
command: 'pnpm',
|
||||
args: ['--filter', filter, script],
|
||||
cwd: workspaceRoot,
|
||||
env,
|
||||
});
|
||||
}
|
||||
commands.push(
|
||||
buildTurboReleaseCommand(workspaceRoot, cacheAnchorRoot, ['@sammo-ts/game-api', '@sammo-ts/gateway-api'], env)
|
||||
);
|
||||
return commands;
|
||||
};
|
||||
|
||||
@@ -1061,7 +1048,12 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
const manifest = await readReleaseManifest(workspace.root);
|
||||
assertReleaseComponents(manifest, ['game-api', 'game-engine', 'game-frontend']);
|
||||
const commands = [
|
||||
...buildWorkspaceCommands(workspace.root, workspace.needsInstall, this.processConfig.baseEnv),
|
||||
...buildWorkspaceCommands(
|
||||
workspace.root,
|
||||
workspace.needsInstall,
|
||||
this.processConfig.baseEnv,
|
||||
this.processConfig.workspaceRoot
|
||||
),
|
||||
...buildProfileFrontendCommands(workspace.root, profile, this.processConfig.baseEnv),
|
||||
];
|
||||
const result = await this.buildRunner.run(commands);
|
||||
@@ -1552,7 +1544,12 @@ export class GatewayOrchestrator implements GatewayOrchestratorHandle {
|
||||
}> {
|
||||
const workspace = await this.workspaceManager.prepare(commitSha);
|
||||
const commands = [
|
||||
...buildWorkspaceCommands(workspace.root, workspace.needsInstall, this.processConfig.baseEnv),
|
||||
...buildWorkspaceCommands(
|
||||
workspace.root,
|
||||
workspace.needsInstall,
|
||||
this.processConfig.baseEnv,
|
||||
this.processConfig.workspaceRoot
|
||||
),
|
||||
...(profile ? buildProfileFrontendCommands(workspace.root, profile, this.processConfig.baseEnv) : []),
|
||||
];
|
||||
return { result: await this.buildRunner.run(commands), workspace };
|
||||
|
||||
@@ -2,7 +2,54 @@ import path from 'node:path';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { MAX_BUILD_OUTPUT_CHARS, PnpmBuildRunner } from '../src/orchestrator/buildRunner.js';
|
||||
import {
|
||||
buildTurboReleaseCommand,
|
||||
MAX_BUILD_OUTPUT_CHARS,
|
||||
PnpmBuildRunner,
|
||||
resolveReleaseTurboCacheDir,
|
||||
} from '../src/orchestrator/buildRunner.js';
|
||||
|
||||
describe('Turbo release build plan', () => {
|
||||
it('anchors the default cache outside commit worktrees and allows an operator override', () => {
|
||||
expect(resolveReleaseTurboCacheDir('/srv/core/repository')).toBe('/srv/core/repository/.turbo/release-cache');
|
||||
expect(
|
||||
resolveReleaseTurboCacheDir('/srv/core/repository', {
|
||||
TURBO_CACHE_DIR: '/srv/core/cache/turbo',
|
||||
})
|
||||
).toBe('/srv/core/cache/turbo');
|
||||
expect(
|
||||
resolveReleaseTurboCacheDir('/srv/core/repository', {
|
||||
TURBO_CACHE_DIR: '.cache/turbo',
|
||||
})
|
||||
).toBe('/srv/core/repository/.cache/turbo');
|
||||
});
|
||||
|
||||
it('uses a bounded streaming Turbo build for the selected packages', () => {
|
||||
expect(
|
||||
buildTurboReleaseCommand(
|
||||
'/srv/core/profile-worktrees/commit',
|
||||
'/srv/core/repository',
|
||||
['@sammo-ts/game-api'],
|
||||
{ NODE_ENV: 'production' }
|
||||
)
|
||||
).toEqual({
|
||||
command: 'pnpm',
|
||||
args: [
|
||||
'exec',
|
||||
'turbo',
|
||||
'run',
|
||||
'build',
|
||||
'--filter=@sammo-ts/game-api',
|
||||
'--cache-dir=/srv/core/repository/.turbo/release-cache',
|
||||
'--concurrency=2',
|
||||
'--ui=stream',
|
||||
'--output-logs=new-only',
|
||||
],
|
||||
cwd: '/srv/core/profile-worktrees/commit',
|
||||
env: { NODE_ENV: 'production' },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('PnpmBuildRunner', () => {
|
||||
it('returns a failed result when a command cannot be spawned', async () => {
|
||||
|
||||
@@ -225,17 +225,22 @@ describe('sanitizeManagedProcessEnv', () => {
|
||||
describe('buildWorkspaceCommands', () => {
|
||||
it('installs and builds runtime dependencies before the profile processes', () => {
|
||||
const workspaceRoot = '/srv/sammo/worktrees/0123456789abcdef';
|
||||
const commands = buildWorkspaceCommands(workspaceRoot, true);
|
||||
const commands = buildWorkspaceCommands(workspaceRoot, true, undefined, '/srv/sammo/controller');
|
||||
|
||||
expect(commands.map(({ args }) => args)).toEqual([
|
||||
['install', '--frozen-lockfile'],
|
||||
['--filter', '@sammo-ts/common', 'build'],
|
||||
['--filter', '@sammo-ts/infra', 'prisma:generate'],
|
||||
['--filter', '@sammo-ts/infra', 'build'],
|
||||
['--filter', '@sammo-ts/logic', 'build'],
|
||||
['--filter', '@sammo-ts/game-api', 'build'],
|
||||
['--filter', '@sammo-ts/game-engine', 'build'],
|
||||
['--filter', '@sammo-ts/gateway-api', 'build'],
|
||||
[
|
||||
'exec',
|
||||
'turbo',
|
||||
'run',
|
||||
'build',
|
||||
'--filter=@sammo-ts/game-api',
|
||||
'--filter=@sammo-ts/gateway-api',
|
||||
'--cache-dir=/srv/sammo/controller/.turbo/release-cache',
|
||||
'--concurrency=2',
|
||||
'--ui=stream',
|
||||
'--output-logs=new-only',
|
||||
],
|
||||
]);
|
||||
expect(commands.every(({ cwd }) => cwd === workspaceRoot)).toBe(true);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user