diff --git a/.env.example b/.env.example index 6430c2a..918a359 100644 --- a/.env.example +++ b/.env.example @@ -20,11 +20,22 @@ RUNTIME_MEMORY_LIMIT=4g RUNTIME_MEMORY_SWAP_LIMIT=4g RUNTIME_CPU_LIMIT=4 RUNTIME_PIDS_LIMIT=512 -# Bound Node/Rolldown build parallelism inside the runtime container. +# Runtime Node limits apply to APIs, daemons, and workers. RUNTIME_NODE_OPTIONS=--max-old-space-size=1536 # Optional turn-daemon-only heap. Raise the runtime memory hard limit first. TURN_DAEMON_NODE_OPTIONS=--max-old-space-size=1536 RUNTIME_RAYON_NUM_THREADS=1 + +# Isolated release builder. It has no DB, Redis, game-token, OAuth, or Docker +# socket credentials and serializes validated jobs on the private network. +BUILDER_MEMORY_LIMIT=4g +BUILDER_MEMORY_SWAP_LIMIT=4g +BUILDER_CPU_LIMIT=4 +BUILDER_PIDS_LIMIT=256 +BUILDER_NODE_OPTIONS=--max-old-space-size=3072 +PROFILE_FRONTEND_BUILD_NODE_OPTIONS=--max-old-space-size=3072 +BUILDER_RAYON_NUM_THREADS=2 +RELEASE_TURBO_CONCURRENCY=1 # Optional smoke-only limits used with compose.smoke.yaml (maximum supported defaults shown). # SMOKE_RUNTIME_MEMORY_LIMIT=4g # SMOKE_RUNTIME_MEMORY_SWAP_LIMIT=4g diff --git a/README.md b/README.md index 2c033da..e879d7a 100644 --- a/README.md +++ b/README.md @@ -67,18 +67,18 @@ Core2026 clone, frozen-lockfile install, build와 Gateway migration 때문에 `Host` header를 보존해야 합니다. runtime은 기본적으로 memory/swap 각각 4 GiB, CPU 4개, PID 512개 상한을 -가집니다. Gateway와 여러 profile의 PM2 process가 실행 중인 상태에서도 release -build의 Node/Rolldown thread를 수용하기 위한 값입니다. 호스트 용량에 맞춰 +가집니다. 이 상한은 Gateway와 여러 profile의 PM2 backend process에만 적용되며 +release build는 별도 builder service의 상한을 사용합니다. 호스트 용량에 맞춰 `RUNTIME_*_LIMIT`을 조정할 수 있지만 0 또는 무제한으로 두지 않습니다. PM2 process/restart 수가 예상보다 증가하면 로그 수집보다 runtime 중지가 우선입니다. -초기 빌드와 profile worktree 빌드는 기본 Node heap 1536 MiB, -`RAYON_NUM_THREADS=1` 안에서 실행됩니다. `RUNTIME_NODE_OPTIONS`의 heap 상한은 +runtime Node process는 기본 heap 1536 MiB, `RAYON_NUM_THREADS=1` 안에서 +실행됩니다. `RUNTIME_NODE_OPTIONS`의 heap 상한은 512~2048 MiB, `RUNTIME_RAYON_NUM_THREADS`는 1~4만 허용됩니다. 턴 데몬만 더 큰 heap이 필요하면 `TURN_DAEMON_NODE_OPTIONS`를 512~4096 MiB 범위에서 지정합니다. 이 값은 Gateway orchestrator가 생성하는 turn-daemon PM2 process에만 -`NODE_OPTIONS`로 적용되며 API·frontend·worker와 build는 계속 -`RUNTIME_NODE_OPTIONS`를 사용합니다. 큰 값을 쓰기 전에 `RUNTIME_MEMORY_LIMIT`과 +`NODE_OPTIONS`로 적용되며 API·worker는 계속 `RUNTIME_NODE_OPTIONS`를 +사용합니다. 큰 값을 쓰기 전에 `RUNTIME_MEMORY_LIMIT`과 동일한 swap 상한을 함께 늘리고 실제 container 총사용량을 확인합니다. `scripts/check.sh`는 example placeholder, 짧은 비밀값, 잘못된 domain/email, @@ -142,7 +142,7 @@ mode 0600 secret 파일에만 둡니다. ```sh docker compose ps -docker compose logs --tail=200 runtime caddy +docker compose logs --tail=200 builder runtime caddy docker compose exec runtime pnpm --filter @sammo-ts/release-controller status ``` @@ -160,12 +160,63 @@ Gateway와 profile frontend의 Vite build는 `/gateway/assets/`, 재사용할 수 있습니다. `index.html`, router fallback, `terms.*.html`처럼 URL이 고정된 HTML은 이 정책에 -포함하지 않습니다. 이 응답은 Vite preview의 `Cache-Control: no-cache`와 ETag를 -유지하여 저장은 허용하되 사용할 때마다 변경 여부를 재검증합니다. content hash가 +포함하지 않습니다. Caddy는 이 응답에 `Cache-Control: no-cache`를 적용하여 +저장은 허용하되 사용할 때마다 변경 여부를 재검증합니다. content hash가 없는 `/assets/` 파일과 별도 운영 자산인 `/image/*`도 `immutable`로 취급하지 않습니다. 따라서 public directory에 장기 cache할 파일을 추가할 때는 먼저 content-hashed URL로 옮겨야 합니다. +## 정적 frontend와 격리 builder + +운영 frontend는 `vite preview`로 제공하지 않습니다. Gateway와 profile build는 +`frontend-artifacts` named volume의 commit/digest 기반 불변 디렉터리에 복사되고, +runtime은 검증된 release의 `current` 심볼릭 링크만 원자적으로 교체합니다. Caddy는 +이 volume을 read-only로 mount하여 직접 제공합니다. 첫 전환과 명시적 개발 모드를 +위한 preview reverse-proxy fallback은 남아 있지만 운영 PM2에는 Vite process가 +없습니다. + +`builder` service는 Core source/worktree volume만 공유하고 release build를 한 번에 +하나씩 실행합니다. DB, Redis, game token, OAuth 비밀값과 Docker socket은 받지 +않으며, runtime은 공개 `VITE_*`와 heap/thread/cache 설정만 build 요청에 +전달합니다. Migration, profile seed, Redis mutation, PM2 전환, readiness와 release +상태 게시는 계속 runtime이 소유합니다. + +release 순서는 builder build → 불변 frontend stage → migration/seed → backend PM2 +전환 → `current` 활성화 → API/Caddy readiness → 상태 게시입니다. 실패하면 이전 +backend와 artifact를 함께 복구합니다. `DEPLOY`는 DB와 현 시즌을 보존하며, +`RESET`은 명시적으로 시나리오를 초기화한 뒤 같은 선택 commit의 artifact를 +게시합니다. `STOPPED`와 가오픈 전 `RESERVED`는 `current`를 제거하여 오래된 SPA가 +계속 노출되지 않게 합니다. + +성공한 Gateway release는 Core clone의 `refs/sammo/active-gateway`도 compare-and-swap으로 +갱신합니다. Runtime container가 재생성되면 tracked 변경이 없는지 확인한 뒤 이 ref를 +detached checkout하여, release worktree에서 성공한 UPDATE가 예전 bootstrap HEAD로 +되돌아가지 않게 합니다. Ref가 아직 없는 최초 설치만 `CORE_BOOTSTRAP_REF` checkout을 +사용합니다. DB 상태 게시가 실패하면 ref도 이전 commit으로 복구합니다. + +기존 preview stack을 처음 전환할 때는 Core main 반영 뒤 현재 runtime 안에서 +release-controller를 먼저 새 commit으로 self-upgrade하고, 그 controller로 Gateway +UPDATE를 완료하여 위 persistent ref가 생성된 것을 확인한 다음 Docker stack을 +재생성합니다. 이 순서를 건너뛰고 새 Compose부터 적용하면 기존 Core bootstrap +checkout에는 artifact publisher가 없을 수 있습니다. 이후 일반 Gateway UPDATE와 +rollback은 controller가 ref를 함께 관리하므로 같은 사전 절차를 반복하지 않습니다. + +```sh +docker compose exec \ + -e GATEWAY_ACTIVE_RELEASE_GIT_REF=refs/sammo/active-gateway \ + runtime pnpm --filter @sammo-ts/release-controller self-upgrade BRANCH main +``` + +이 명령 뒤 관리자 Gateway UPDATE가 성공하고 ref와 active commit이 같은지 확인합니다. +현재 RUNNING/PREOPEN profile은 Docker 재생성 전에 같은 commit으로 DB 보존 DEPLOY하여 +`.release-dist`를 준비합니다. RESET은 DB 폐기 의도가 있는 dev 대상에서 별도로 +검증하며, 정적 제공 전환 자체를 위해 운영 DB를 RESET하지 않습니다. + +`frontend-artifacts`와 `builder-cache`, Core clone 안의 active release ref도 일반 +`down`에서는 보존됩니다. +`down --volumes`는 DB뿐 아니라 이 release 상태도 삭제하므로 별도 폐기 권한과 +backup 없이는 실행하지 않습니다. + ## 개발 bind 모드 로컬 Core2026 checkout을 container에 bind하고 DB/Redis/Caddy는 같은 구성으로 diff --git a/caddy/Caddyfile b/caddy/Caddyfile index 51f0ce9..ab00d27 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -7,13 +7,15 @@ } } -{$SITE_ADDRESS} { +{$SITE_ADDRESS}, http://caddy { encode zstd gzip # Vite emits content-hashed files below each frontend's assets directory. # Defer this header so it replaces vite preview's default no-cache value. @immutableFrontendAssets path_regexp immutableFrontendAssets ^/(gateway|che|kwe|pwe|twe|nya|pya|hwe)/assets/.+-[A-Za-z0-9_-]{8}\.[^/]+$ header @immutableFrontendAssets >Cache-Control "public, max-age=31536000, immutable" + @frontendRequests path /gateway/* /che/* /kwe/* /pwe/* /twe/* /nya/* /pya/* /hwe/* + header @frontendRequests Cache-Control "no-cache" redir /gateway /gateway/ 308 redir /che /che/ 308 @@ -41,6 +43,24 @@ reverse_proxy runtime:15001 } handle /gateway/* { + @gatewayPublished file { + root /srv/frontend-artifacts/gateway/current + try_files /index.html + } + handle @gatewayPublished { + route { + uri strip_prefix /gateway + root * /srv/frontend-artifacts/gateway/current + @gatewayAsset path /assets/* + handle @gatewayAsset { + file_server + } + handle { + try_files {path} /index.html + file_server + } + } + } reverse_proxy runtime:15000 } @@ -48,42 +68,168 @@ reverse_proxy runtime:15003 } handle /che/* { + @chePublished file { + root /srv/frontend-artifacts/che/current + try_files /index.html + } + handle @chePublished { + route { + uri strip_prefix /che + root * /srv/frontend-artifacts/che/current + @cheAsset path /assets/* + handle @cheAsset { + file_server + } + handle { + try_files {path} /index.html + file_server + } + } + } reverse_proxy runtime:15002 } handle /kwe/api/* { reverse_proxy runtime:15005 } handle /kwe/* { + @kwePublished file { + root /srv/frontend-artifacts/kwe/current + try_files /index.html + } + handle @kwePublished { + route { + uri strip_prefix /kwe + root * /srv/frontend-artifacts/kwe/current + @kweAsset path /assets/* + handle @kweAsset { + file_server + } + handle { + try_files {path} /index.html + file_server + } + } + } reverse_proxy runtime:15004 } handle /pwe/api/* { reverse_proxy runtime:15007 } handle /pwe/* { + @pwePublished file { + root /srv/frontend-artifacts/pwe/current + try_files /index.html + } + handle @pwePublished { + route { + uri strip_prefix /pwe + root * /srv/frontend-artifacts/pwe/current + @pweAsset path /assets/* + handle @pweAsset { + file_server + } + handle { + try_files {path} /index.html + file_server + } + } + } reverse_proxy runtime:15006 } handle /twe/api/* { reverse_proxy runtime:15009 } handle /twe/* { + @twePublished file { + root /srv/frontend-artifacts/twe/current + try_files /index.html + } + handle @twePublished { + route { + uri strip_prefix /twe + root * /srv/frontend-artifacts/twe/current + @tweAsset path /assets/* + handle @tweAsset { + file_server + } + handle { + try_files {path} /index.html + file_server + } + } + } reverse_proxy runtime:15008 } handle /nya/api/* { reverse_proxy runtime:15011 } handle /nya/* { + @nyaPublished file { + root /srv/frontend-artifacts/nya/current + try_files /index.html + } + handle @nyaPublished { + route { + uri strip_prefix /nya + root * /srv/frontend-artifacts/nya/current + @nyaAsset path /assets/* + handle @nyaAsset { + file_server + } + handle { + try_files {path} /index.html + file_server + } + } + } reverse_proxy runtime:15010 } handle /pya/api/* { reverse_proxy runtime:15013 } handle /pya/* { + @pyaPublished file { + root /srv/frontend-artifacts/pya/current + try_files /index.html + } + handle @pyaPublished { + route { + uri strip_prefix /pya + root * /srv/frontend-artifacts/pya/current + @pyaAsset path /assets/* + handle @pyaAsset { + file_server + } + handle { + try_files {path} /index.html + file_server + } + } + } reverse_proxy runtime:15012 } handle /hwe/api/* { reverse_proxy runtime:15015 } handle /hwe/* { + @hwePublished file { + root /srv/frontend-artifacts/hwe/current + try_files /index.html + } + handle @hwePublished { + route { + uri strip_prefix /hwe + root * /srv/frontend-artifacts/hwe/current + @hweAsset path /assets/* + handle @hweAsset { + file_server + } + handle { + try_files {path} /index.html + file_server + } + } + } reverse_proxy runtime:15014 } diff --git a/compose.dev.yaml b/compose.dev.yaml index 9186d4e..e307fdd 100644 --- a/compose.dev.yaml +++ b/compose.dev.yaml @@ -3,12 +3,20 @@ services: restart: "no" redis: restart: "no" + builder: + user: ${DEV_UID:-1000}:${DEV_GID:-1000} + volumes: + - ${CORE_DEV_PATH:-../core2026}:/workspace/core2026 + - core-source:/srv/core + - builder-cache:/srv/builder-cache + restart: "no" runtime: user: ${DEV_UID:-1000}:${DEV_GID:-1000} environment: CORE_SOURCE_MODE: bind CORE_BIND_ROOT: /workspace/core2026 RUNTIME_MODE: development + FRONTEND_SERVE_MODE: preview VITE_PREVIEW_ALLOWED_HOSTS: '*' volumes: - ${CORE_DEV_PATH:-../core2026}:/workspace/core2026 diff --git a/compose.e2e.yaml b/compose.e2e.yaml index a08c928..84daa0a 100644 --- a/compose.e2e.yaml +++ b/compose.e2e.yaml @@ -3,6 +3,12 @@ services: restart: "no" redis: restart: "no" + builder: + volumes: + - /home/letrhee/sam_rebuild/core2026:/workspace/core2026 + - core-source:/srv/core + - builder-cache:/srv/builder-cache + restart: "no" runtime: environment: CORE_SOURCE_MODE: bind diff --git a/compose.smoke.yaml b/compose.smoke.yaml index bd9ffd9..17ba765 100644 --- a/compose.smoke.yaml +++ b/compose.smoke.yaml @@ -3,6 +3,8 @@ services: restart: "no" redis: restart: "no" + builder: + restart: "no" runtime: mem_limit: ${SMOKE_RUNTIME_MEMORY_LIMIT:-4g} memswap_limit: ${SMOKE_RUNTIME_MEMORY_SWAP_LIMIT:-4g} diff --git a/compose.yaml b/compose.yaml index 7cc7331..53d4848 100644 --- a/compose.yaml +++ b/compose.yaml @@ -34,6 +34,37 @@ services: start_period: 5s restart: unless-stopped + builder: + build: + context: . + dockerfile: runtime/Dockerfile + entrypoint: [/usr/bin/tini, --, node, /opt/sammo/release-builder.mjs] + environment: + CI: 'true' + NODE_OPTIONS: ${BUILDER_NODE_OPTIONS:---max-old-space-size=3072} + PROFILE_FRONTEND_BUILD_NODE_OPTIONS: ${PROFILE_FRONTEND_BUILD_NODE_OPTIONS:---max-old-space-size=3072} + RAYON_NUM_THREADS: ${BUILDER_RAYON_NUM_THREADS:-2} + RELEASE_TURBO_CONCURRENCY: ${RELEASE_TURBO_CONCURRENCY:-1} + TURBO_CACHE_DIR: /srv/builder-cache/turbo + XDG_DATA_HOME: /srv/builder-cache/xdg + RELEASE_BUILDER_PORT: '15100' + TZ: ${TZ:-Asia/Seoul} + volumes: + - core-source:/srv/core + - builder-cache:/srv/builder-cache + healthcheck: + test: [CMD, curl, --fail, --silent, http://127.0.0.1:15100/healthz] + interval: 5s + timeout: 5s + retries: 24 + start_period: 5s + mem_limit: ${BUILDER_MEMORY_LIMIT:-4g} + memswap_limit: ${BUILDER_MEMORY_SWAP_LIMIT:-4g} + cpus: ${BUILDER_CPU_LIMIT:-4} + pids_limit: ${BUILDER_PIDS_LIMIT:-256} + stop_grace_period: 15s + restart: unless-stopped + runtime: build: context: . @@ -89,6 +120,11 @@ services: GATEWAY_API_HOST: 0.0.0.0 GATEWAY_API_PORT: '15001' GATEWAY_FRONTEND_PORT: '15000' + FRONTEND_SERVE_MODE: static + FRONTEND_ARTIFACT_ROOT: /srv/frontend-artifacts + FRONTEND_READINESS_ORIGIN: http://caddy + RELEASE_BUILDER_URL: http://builder:15100 + RELEASE_BUILD_NODE_OPTIONS: ${BUILDER_NODE_OPTIONS:---max-old-space-size=3072} GATEWAY_TRPC_PATH: /gateway/api/trpc CORE_NAVIGATION_CONFIG_FILE: /srv/data/navigation.json GATEWAY_INTERNAL_API_URL: http://127.0.0.1:15001 @@ -97,6 +133,7 @@ services: GATEWAY_WORKTREE_ROOT: /srv/core/profile-worktrees RELEASE_CONTROLLER_WORKSPACE_ROOT: /srv/core/repository RELEASE_CONTROLLER_WORKTREE_ROOT: /srv/core/release-worktrees + GATEWAY_ACTIVE_RELEASE_GIT_REF: refs/sammo/active-gateway GATEWAY_BASE_PATH: /gateway VITE_APP_BASE_PATH: /gateway VITE_GATEWAY_API_URL: /gateway/api/trpc @@ -110,6 +147,7 @@ services: volumes: - core-source:/srv/core - runtime-data:/srv/data + - frontend-artifacts:/srv/frontend-artifacts - ${IMAGE_UPLOAD_CORE2026_SECRET_FILE:?set IMAGE_UPLOAD_CORE2026_SECRET_FILE in .env}:/run/secrets/image_upload_core2026_secret:ro - ${IMAGE_SYNC_CORE2026_SECRET_FILE:?set IMAGE_SYNC_CORE2026_SECRET_FILE in .env}:/run/secrets/image_sync_core2026_secret:ro depends_on: @@ -117,6 +155,8 @@ services: condition: service_healthy redis: condition: service_healthy + builder: + condition: service_healthy healthcheck: test: [CMD, curl, --fail, --silent, http://127.0.0.1:15001/healthz] interval: 10s @@ -143,6 +183,7 @@ services: volumes: - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro - ./data/image:/srv/image:ro + - frontend-artifacts:/srv/frontend-artifacts:ro - caddy-data:/data - caddy-config:/config depends_on: @@ -155,5 +196,7 @@ volumes: redis-data: core-source: runtime-data: + frontend-artifacts: + builder-cache: caddy-data: caddy-config: diff --git a/runtime/Dockerfile b/runtime/Dockerfile index 74b746e..0857882 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -4,12 +4,13 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl git openssh-client procps tini \ && rm -rf /var/lib/apt/lists/* \ && corepack enable \ - && corepack prepare pnpm@11.17.0 --activate + && corepack prepare pnpm@11.21.0 --activate COPY --chmod=755 runtime/entrypoint.sh /usr/local/bin/sammo-entrypoint COPY --chmod=755 runtime/redis-entrypoint.sh /opt/sammo/redis-entrypoint.sh COPY --chmod=755 runtime/git-askpass.sh /opt/sammo/git-askpass.sh -COPY runtime/ecosystem.config.cjs runtime/bootstrap.mjs runtime/validate-env.mjs /opt/sammo/ +COPY --chmod=755 runtime/checkout-active-release.sh /opt/sammo/checkout-active-release.sh +COPY runtime/ecosystem.config.cjs runtime/bootstrap.mjs runtime/validate-env.mjs runtime/release-builder.mjs runtime/release-builder-client.mjs /opt/sammo/ WORKDIR /srv/core/repository ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/sammo-entrypoint"] diff --git a/runtime/checkout-active-release.sh b/runtime/checkout-active-release.sh new file mode 100644 index 0000000..af6c382 --- /dev/null +++ b/runtime/checkout-active-release.sh @@ -0,0 +1,31 @@ +#!/bin/sh +set -eu + +core_root=${1:?usage: checkout-active-release.sh CORE_ROOT RELEASE_REF} +active_release_ref=${2:-} + +if [ -z "$active_release_ref" ]; then + exit 0 +fi +case "$active_release_ref" in + refs/sammo/*) ;; + *) + echo 'GATEWAY_ACTIVE_RELEASE_GIT_REF must be below refs/sammo/' >&2 + exit 64 + ;; +esac +if ! git check-ref-format "$active_release_ref" >/dev/null 2>&1; then + echo 'GATEWAY_ACTIVE_RELEASE_GIT_REF is not a valid Git ref' >&2 + exit 64 +fi +if ! git -C "$core_root" show-ref --verify --quiet "$active_release_ref"; then + exit 0 +fi +if ! git -C "$core_root" diff --quiet || ! git -C "$core_root" diff --cached --quiet; then + echo 'Core bootstrap checkout has tracked changes; refusing to switch the active release ref' >&2 + exit 65 +fi + +active_release_sha=$(git -C "$core_root" rev-parse --verify "${active_release_ref}^{commit}") +git -C "$core_root" checkout --quiet --detach "$active_release_sha" +echo "Using persistent Gateway release $active_release_sha for runtime bootstrap." diff --git a/runtime/ecosystem.config.cjs b/runtime/ecosystem.config.cjs index 192c3eb..a5878ac 100644 --- a/runtime/ecosystem.config.cjs +++ b/runtime/ecosystem.config.cjs @@ -19,13 +19,6 @@ module.exports = { script: path.join(root, 'app/gateway-api/dist/index.js'), env: { ...process.env, GATEWAY_ROLE: 'api' }, }, - { - ...common, - name: 'sammo:gateway-frontend', - cwd: path.join(root, 'app/gateway-frontend'), - script: path.join(root, 'app/gateway-frontend/node_modules/vite/bin/vite.js'), - args: 'preview --host 0.0.0.0 --port 15000', - }, { ...common, name: 'sammo:gateway-orchestrator', diff --git a/runtime/entrypoint.sh b/runtime/entrypoint.sh index 019912c..1d8c73f 100755 --- a/runtime/entrypoint.sh +++ b/runtime/entrypoint.sh @@ -69,6 +69,7 @@ case "${CORE_SOURCE_MODE:-clone}" in esac git config --global --add safe.directory "$core_root" +/opt/sammo/checkout-active-release.sh "$core_root" "${GATEWAY_ACTIVE_RELEASE_GIT_REF:-}" export GATEWAY_WORKSPACE_ROOT="$core_root" export RELEASE_CONTROLLER_WORKSPACE_ROOT="$core_root" @@ -85,7 +86,6 @@ export GATEWAY_DATABASE_URL="$DATABASE_URL" export REDIS_URL="$(node -e 'const u=new URL("redis://localhost/0");u.password=process.env.REDIS_PASSWORD;u.hostname=process.env.REDIS_HOST||"redis";u.port=process.env.REDIS_PORT||"6379";process.stdout.write(u.href)')" cd "$core_root" -pnpm install --frozen-lockfile if [ "${RUNTIME_MODE:-production}" = development ]; then pnpm --filter @sammo-ts/infra prisma:generate @@ -96,16 +96,16 @@ if [ "${RUNTIME_MODE:-production}" = development ]; then exec sleep infinity fi -pnpm --filter @sammo-ts/common build -pnpm --filter @sammo-ts/infra prisma:generate -pnpm --filter @sammo-ts/infra build -pnpm --filter @sammo-ts/logic build -pnpm --filter @sammo-ts/game-engine build -pnpm --filter @sammo-ts/gateway-api build -pnpm --filter @sammo-ts/gateway-frontend build -pnpm --filter @sammo-ts/release-controller build +node /opt/sammo/release-builder-client.mjs pnpm --filter @sammo-ts/infra prisma:migrate:deploy:gateway +core_commit_sha=$(git rev-parse HEAD) +node tools/build-scripts/publish-frontend-artifact.mjs \ + --artifact-root "${FRONTEND_ARTIFACT_ROOT:-/srv/frontend-artifacts}" \ + --frontend-key gateway \ + --source-root "$core_root/app/gateway-frontend/dist" \ + --commit-sha "$core_commit_sha" + GATEWAY_ROLE=api node app/gateway-api/dist/index.js & bootstrap_pid=$! cleanup_bootstrap() { diff --git a/runtime/release-builder-client.mjs b/runtime/release-builder-client.mjs new file mode 100644 index 0000000..3bc355d --- /dev/null +++ b/runtime/release-builder-client.mjs @@ -0,0 +1,57 @@ +const builderUrl = process.env.RELEASE_BUILDER_URL ?? 'http://builder:15100'; +const coreRoot = process.env.GATEWAY_WORKSPACE_ROOT ?? '/srv/core/repository'; +const publicBuildEnv = Object.fromEntries( + Object.entries(process.env).filter( + ([name, value]) => + typeof value === 'string' && + /^(?:CI|NODE_OPTIONS|PROFILE_FRONTEND_BUILD_NODE_OPTIONS|RAYON_NUM_THREADS|RELEASE_TURBO_CONCURRENCY|TURBO_CACHE_DIR|TZ|VITE_[A-Z0-9_]+)$/.test(name), + ), +); +if (process.env.RELEASE_BUILD_NODE_OPTIONS) { + publicBuildEnv.NODE_OPTIONS = process.env.RELEASE_BUILD_NODE_OPTIONS; +} +delete publicBuildEnv.RELEASE_BUILD_NODE_OPTIONS; +const pnpm = (...args) => ({ command: 'pnpm', args, cwd: coreRoot, env: publicBuildEnv }); +const commands = [ + pnpm('install', '--frozen-lockfile'), + pnpm('--filter', '@sammo-ts/common', 'build'), + pnpm('--filter', '@sammo-ts/infra', 'prisma:generate'), + pnpm('--filter', '@sammo-ts/infra', 'build'), + pnpm('--filter', '@sammo-ts/logic', 'build'), + pnpm('--filter', '@sammo-ts/game-engine', 'build'), + pnpm('--filter', '@sammo-ts/gateway-api', 'build'), + pnpm('--filter', '@sammo-ts/gateway-frontend', 'build'), + pnpm('--filter', '@sammo-ts/release-controller', 'build'), +]; + +const response = await fetch(new URL('/v1/builds', builderUrl), { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ commands }), +}); +if (!response.ok || !response.body) { + throw new Error((await response.text()) || `Release builder returned HTTP ${response.status}.`); +} +const decoder = new TextDecoder(); +let buffer = ''; +let result = null; +const handleLine = (line) => { + if (!line.trim()) return; + const message = JSON.parse(line); + if (message.event?.type === 'OUTPUT') { + const output = `${message.event.message}\n`; + if (message.event.stream === 'stderr') process.stderr.write(output); + else process.stdout.write(output); + } + if (message.result) result = message.result; +}; +for await (const chunk of response.body) { + buffer += decoder.decode(chunk, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + for (const line of lines) handleLine(line); +} +handleLine(buffer); +if (!result?.ok) { + throw new Error(result?.output || 'Release builder closed without a successful result.'); +} diff --git a/runtime/release-builder.mjs b/runtime/release-builder.mjs new file mode 100644 index 0000000..e6a3613 --- /dev/null +++ b/runtime/release-builder.mjs @@ -0,0 +1,182 @@ +import { spawn } from 'node:child_process'; +import http from 'node:http'; +import path from 'node:path'; + +const port = Number(process.env.RELEASE_BUILDER_PORT ?? 15100); +const allowedRoots = [ + '/srv/core/repository', + '/srv/core/profile-worktrees', + '/srv/core/release-worktrees', + '/workspace/core2026', +].map((root) => path.resolve(root)); +const allowedEnvName = /^(?:CI|NODE_OPTIONS|PROFILE_FRONTEND_BUILD_NODE_OPTIONS|RAYON_NUM_THREADS|RELEASE_TURBO_CONCURRENCY|TURBO_CACHE_DIR|TZ|VITE_[A-Z0-9_]+)$/; +const maxOutput = 64 * 1024; +let queue = Promise.resolve(); + +const appendTail = (current, chunk) => `${current}${String(chunk)}`.slice(-maxOutput); + +const isWithin = (candidate, root) => { + const relative = path.relative(root, candidate); + return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); +}; + +const validateCommand = (raw) => { + if (!raw || typeof raw !== 'object' || !Array.isArray(raw.args) || typeof raw.cwd !== 'string') { + throw new Error('Each release build command must provide command, args, and cwd.'); + } + const cwd = path.resolve(raw.cwd); + if (!allowedRoots.some((root) => isWithin(cwd, root))) { + throw new Error(`Release build cwd is outside the shared Core workspaces: ${cwd}`); + } + const args = raw.args.map((value) => String(value)); + if (raw.command === 'pnpm') { + const install = args[0] === 'install' && args.includes('--frozen-lockfile'); + const turbo = args[0] === 'exec' && args[1] === 'turbo' && args[2] === 'run'; + const packageTask = args[0] === '--filter' && args.length >= 3 && ['build', 'prisma:generate'].includes(args[2]); + if (!install && !turbo && !packageTask) throw new Error('Unsupported pnpm release build command.'); + } else if (raw.command === 'node') { + if (args[0] !== 'tools/build-scripts/materialize-profile-frontend.mjs' || args.length !== 2) { + throw new Error('Unsupported Node release build command.'); + } + } else { + throw new Error(`Unsupported release build executable: ${raw.command}`); + } + const overrides = Object.fromEntries( + Object.entries(raw.env ?? {}).filter( + ([name, value]) => allowedEnvName.test(name) && typeof value === 'string', + ), + ); + return { + command: raw.command, + args, + cwd, + env: { ...process.env, ...overrides }, + eventCommand: { command: raw.command, args, cwd, env: overrides }, + }; +}; + +const terminate = (child, signal) => { + if (!child.pid) return; + try { + process.kill(-child.pid, signal); + } catch { + try { + child.kill(signal); + } catch { + // The build already exited. + } + } +}; + +const runCommand = (command, emit, signal) => + new Promise((resolve) => { + emit({ event: { type: 'COMMAND_START', command: command.eventCommand } }); + const child = spawn(command.command, command.args, { + cwd: command.cwd, + env: command.env, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true, + }); + let output = ''; + let aborted = false; + let spawnError = null; + const abort = () => { + if (aborted) return; + aborted = true; + terminate(child, 'SIGTERM'); + const killTimer = setTimeout(() => terminate(child, 'SIGKILL'), 5000); + killTimer.unref(); + }; + signal.addEventListener('abort', abort, { once: true }); + for (const stream of ['stdout', 'stderr']) { + child[stream].on('data', (chunk) => { + output = appendTail(output, chunk); + for (const message of String(chunk).split(/\r?\n/).filter(Boolean)) { + emit({ event: { type: 'OUTPUT', stream, message: message.slice(0, 2000) } }); + } + }); + } + child.on('error', (error) => { + spawnError = error; + output = appendTail(output, error.message); + }); + child.on('close', (exitCode) => { + signal.removeEventListener('abort', abort); + emit({ event: { type: 'COMMAND_END', command: command.eventCommand, exitCode: spawnError ? null : exitCode } }); + resolve({ + ok: !aborted && !spawnError && exitCode === 0, + exitCode: spawnError ? null : exitCode, + output, + ...(aborted ? { aborted: true } : {}), + }); + }); + }); + +const runJob = async (commands, emit, signal) => { + let output = ''; + for (const command of commands) { + if (signal.aborted) return { ok: false, exitCode: null, output, aborted: true }; + const result = await runCommand(command, emit, signal); + output = appendTail(output, result.output); + if (!result.ok) return { ...result, output }; + } + return { ok: true, exitCode: 0, output }; +}; + +const readJson = async (request) => { + let body = ''; + for await (const chunk of request) { + body += chunk; + if (body.length > 256 * 1024) throw new Error('Release build request is too large.'); + } + return JSON.parse(body); +}; + +const server = http.createServer(async (request, response) => { + if (request.method === 'GET' && request.url === '/healthz') { + response.writeHead(200, { 'content-type': 'text/plain' }); + response.end('ok'); + return; + } + if (request.method !== 'POST' || request.url !== '/v1/builds') { + response.writeHead(404).end(); + return; + } + try { + const payload = await readJson(request); + if (!Array.isArray(payload.commands) || payload.commands.length === 0 || payload.commands.length > 12) { + throw new Error('Release build request must contain 1-12 commands.'); + } + const commands = payload.commands.map(validateCommand); + response.writeHead(200, { + 'content-type': 'application/x-ndjson', + 'cache-control': 'no-store', + 'x-content-type-options': 'nosniff', + }); + response.flushHeaders(); + const abortController = new AbortController(); + request.once('aborted', () => abortController.abort()); + response.once('close', () => { + if (!response.writableEnded) abortController.abort(); + }); + const emit = (message) => { + if (!response.writableEnded) response.write(`${JSON.stringify(message)}\n`); + }; + const execution = queue.then(() => runJob(commands, emit, abortController.signal)); + queue = execution.then(() => undefined, () => undefined); + const result = await execution; + emit({ result }); + response.end(); + } catch (error) { + if (!response.headersSent) response.writeHead(400, { 'content-type': 'text/plain' }); + response.end(error instanceof Error ? error.message : String(error)); + } +}); + +server.listen(port, '0.0.0.0', () => { + console.log(`[release-builder] listening on ${port}`); +}); + +const shutdown = () => server.close(() => process.exit(0)); +process.once('SIGINT', shutdown); +process.once('SIGTERM', shutdown); diff --git a/runtime/validate-compose-model.mjs b/runtime/validate-compose-model.mjs index 90b919b..a78d4e9 100644 --- a/runtime/validate-compose-model.mjs +++ b/runtime/validate-compose-model.mjs @@ -10,6 +10,10 @@ export const validateComposeModel = (model, mode) => { const errors = []; const runtime = model?.services?.runtime; if (!runtime) return ['runtime service is missing']; + const builder = model?.services?.builder; + const caddy = model?.services?.caddy; + if (!builder) errors.push('builder service is missing'); + if (!caddy) errors.push('caddy service is missing'); const requiredImageEnvironment = { GATEWAY_IMAGE_UPLOAD_SECRET_FILE: '/run/secrets/image_upload_core2026_secret', @@ -41,6 +45,16 @@ export const validateComposeModel = (model, mode) => { errors.push(`${target} must be a read-only bind mount`); } } + const artifactMount = mounts.find((candidate) => candidate?.target === '/srv/frontend-artifacts'); + if (!artifactMount || artifactMount.read_only === true) { + errors.push('/srv/frontend-artifacts must be a writable runtime volume'); + } + const caddyArtifactMount = (Array.isArray(caddy?.volumes) ? caddy.volumes : []).find( + (candidate) => candidate?.target === '/srv/frontend-artifacts', + ); + if (!caddyArtifactMount || caddyArtifactMount.read_only !== true) { + errors.push('/srv/frontend-artifacts must be a read-only Caddy volume'); + } if (!positive(runtime.mem_limit)) errors.push('runtime memory limit must be positive'); if (!positive(runtime.memswap_limit)) errors.push('runtime memory+swap limit must be positive'); @@ -70,6 +84,34 @@ export const validateComposeModel = (model, mode) => { errors.push('runtime Rayon thread count must be between 1 and 4'); } + if (builder) { + if (!positive(builder.mem_limit)) errors.push('builder memory limit must be positive'); + if (!positive(builder.memswap_limit)) errors.push('builder memory+swap limit must be positive'); + if (Number(builder.memswap_limit) !== Number(builder.mem_limit)) { + errors.push('builder memory+swap limit must equal its memory limit'); + } + if (!positive(builder.cpus)) errors.push('builder CPU limit must be positive'); + if (!positive(builder.pids_limit)) errors.push('builder PID limit must be positive'); + const builderNodeOptions = builder.environment?.NODE_OPTIONS ?? ''; + const builderHeapMatch = /(?:^|\s)--max-old-space-size=(\d+)(?:\s|$)/.exec(builderNodeOptions); + const builderHeapMiB = Number(builderHeapMatch?.[1] ?? 0); + if (!builderHeapMatch || builderHeapMiB < 512 || builderHeapMiB > 3584) { + errors.push('builder Node heap limit must be between 512 and 3584 MiB'); + } + if (builderHeapMiB * 1024 * 1024 >= Number(builder.mem_limit)) { + errors.push('builder Node heap limit must stay below the builder memory limit'); + } + const builderRayonThreads = Number(builder.environment?.RAYON_NUM_THREADS ?? 0); + if (!Number.isInteger(builderRayonThreads) || builderRayonThreads < 1 || builderRayonThreads > 4) { + errors.push('builder Rayon thread count must be between 1 and 4'); + } + for (const name of Object.keys(builder.environment ?? {})) { + if (/(?:SECRET|TOKEN|PASSWORD|DATABASE_URL|REDIS_URL)/i.test(name)) { + errors.push(`builder must not receive sensitive environment variable ${name}`); + } + } + } + if (mode === 'development') { if (runtime.environment?.RUNTIME_MODE !== 'development') { errors.push('development runtime mode must be literal development'); @@ -82,9 +124,20 @@ export const validateComposeModel = (model, mode) => { } else if (runtime.environment?.RUNTIME_MODE === 'development') { errors.push(`${mode} runtime must not use development mode`); } + if (mode !== 'development') { + if (runtime.environment?.FRONTEND_SERVE_MODE !== 'static') { + errors.push(`${mode} frontend serve mode must be static`); + } + if (runtime.environment?.RELEASE_BUILDER_URL !== 'http://builder:15100') { + errors.push(`${mode} runtime must use the isolated release builder`); + } + if (runtime.environment?.GATEWAY_ACTIVE_RELEASE_GIT_REF !== 'refs/sammo/active-gateway') { + errors.push(`${mode} runtime must persist the active Gateway release ref`); + } + } if (mode === 'development' || mode === 'smoke') { - for (const name of ['postgres', 'redis', 'runtime', 'caddy']) { + for (const name of ['postgres', 'redis', 'builder', 'runtime', 'caddy']) { if (model?.services?.[name]?.restart !== 'no') errors.push(`${name} ${mode} restart policy must be no`); } if (Number(runtime.mem_limit) > MAX_SAFE_DEV_MEMORY) errors.push(`${mode} runtime memory limit exceeds 4 GiB`); diff --git a/test/caddy-cache-policy.test.mjs b/test/caddy-cache-policy.test.mjs index eac4e95..4993b05 100644 --- a/test/caddy-cache-policy.test.mjs +++ b/test/caddy-cache-policy.test.mjs @@ -25,3 +25,26 @@ test('Caddy caches only content-hashed Vite frontend assets as immutable', async /header @immutableFrontendAssets >Cache-Control "public, max-age=31536000, immutable"/, ); }); + +test('Caddy serves published frontend artifacts directly and keeps a transition fallback', async () => { + const caddyfile = await fs.readFile(new URL('../caddy/Caddyfile', import.meta.url), 'utf8'); + for (const [profile, port] of Object.entries({ + gateway: 15000, + che: 15002, + kwe: 15004, + pwe: 15006, + twe: 15008, + nya: 15010, + pya: 15012, + hwe: 15014, + })) { + assert.match(caddyfile, new RegExp(`/srv/frontend-artifacts/${profile}/current`)); + assert.match(caddyfile, new RegExp(`reverse_proxy runtime:${port}`)); + assert.match( + caddyfile, + new RegExp(`route \\{[\\s\\S]*?@${profile}Asset path /assets/\\*[\\s\\S]*?handle @${profile}Asset`), + ); + } + assert.match(caddyfile, /try_files \{path\} \/index\.html/); + assert.match(caddyfile, /header @frontendRequests Cache-Control "no-cache"/); +}); diff --git a/test/checkout-active-release.test.mjs b/test/checkout-active-release.test.mjs new file mode 100644 index 0000000..44d9551 --- /dev/null +++ b/test/checkout-active-release.test.mjs @@ -0,0 +1,62 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +const script = path.resolve('runtime/checkout-active-release.sh'); +const roots = []; +const git = (cwd, ...args) => + execFileSync('git', args, { + cwd, + encoding: 'utf8', + env: { + ...process.env, + GIT_AUTHOR_NAME: 'Sammo Test', + GIT_AUTHOR_EMAIL: 'sammo-test@example.invalid', + GIT_COMMITTER_NAME: 'Sammo Test', + GIT_COMMITTER_EMAIL: 'sammo-test@example.invalid', + }, + }).trim(); + +const fixture = () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'sammo-active-release-')); + roots.push(root); + git(root, 'init', '-b', 'main'); + fs.writeFileSync(path.join(root, 'version.txt'), 'one\n'); + git(root, 'add', 'version.txt'); + git(root, 'commit', '-m', 'first'); + const first = git(root, 'rev-parse', 'HEAD'); + fs.writeFileSync(path.join(root, 'version.txt'), 'two\n'); + git(root, 'commit', '-am', 'second'); + const second = git(root, 'rev-parse', 'HEAD'); + git(root, 'update-ref', 'refs/sammo/active-gateway', first); + return { root, first, second }; +}; + +test.afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +test('checks out the persistent active release instead of the previous bootstrap HEAD', () => { + const { root, first, second } = fixture(); + assert.equal(git(root, 'rev-parse', 'HEAD'), second); + execFileSync('/bin/sh', [script, root, 'refs/sammo/active-gateway']); + assert.equal(git(root, 'rev-parse', 'HEAD'), first); +}); + +test('rejects refs outside the managed release namespace', () => { + const { root } = fixture(); + const result = spawnSync('/bin/sh', [script, root, 'refs/heads/main'], { encoding: 'utf8' }); + assert.equal(result.status, 64); + assert.match(result.stderr, /refs\/sammo/u); +}); + +test('refuses to hide tracked bootstrap checkout changes', () => { + const { root, second } = fixture(); + fs.writeFileSync(path.join(root, 'version.txt'), 'dirty\n'); + const result = spawnSync('/bin/sh', [script, root, 'refs/sammo/active-gateway'], { encoding: 'utf8' }); + assert.equal(result.status, 65); + assert.equal(git(root, 'rev-parse', 'HEAD'), second); +}); diff --git a/test/ecosystem-config.test.mjs b/test/ecosystem-config.test.mjs index c61b87a..460fcf1 100644 --- a/test/ecosystem-config.test.mjs +++ b/test/ecosystem-config.test.mjs @@ -5,12 +5,13 @@ import { createRequire } from 'node:module'; const require = createRequire(import.meta.url); const ecosystem = require('../runtime/ecosystem.config.cjs'); -test('defines exactly four unique, bounded Gateway processes', () => { +test('defines only backend Gateway processes and no Vite preview process', () => { assert.deepEqual( ecosystem.apps.map((app) => app.name), - ['sammo:gateway-api', 'sammo:gateway-frontend', 'sammo:gateway-orchestrator', 'sammo:release-controller'], + ['sammo:gateway-api', 'sammo:gateway-orchestrator', 'sammo:release-controller'], ); - assert.equal(new Set(ecosystem.apps.map((app) => app.name)).size, 4); + assert.equal(new Set(ecosystem.apps.map((app) => app.name)).size, 3); + assert.equal(ecosystem.apps.some((app) => String(app.args ?? '').includes('preview')), false); for (const app of ecosystem.apps) { assert.equal(app.autorestart, true); assert.equal(app.max_restarts, 5); diff --git a/test/validate-compose-model.test.mjs b/test/validate-compose-model.test.mjs index 353cbfe..a95759d 100644 --- a/test/validate-compose-model.test.mjs +++ b/test/validate-compose-model.test.mjs @@ -20,6 +20,9 @@ const safeRuntime = { IMAGE_SYNC_SECRET_FILE: '/run/secrets/image_sync_core2026_secret', VITE_IMAGE_PUBLIC_URL: 'https://sam-image.hided.net', VITE_GATEWAY_USER_ICON_BASE_URL: 'https://sam-image.hided.net/icons', + FRONTEND_SERVE_MODE: 'static', + RELEASE_BUILDER_URL: 'http://builder:15100', + GATEWAY_ACTIVE_RELEASE_GIT_REF: 'refs/sammo/active-gateway', }, volumes: [ { @@ -28,6 +31,12 @@ const safeRuntime = { target: '/run/secrets/image_upload_core2026_secret', read_only: true, }, + { + type: 'volume', + source: 'frontend-artifacts', + target: '/srv/frontend-artifacts', + read_only: false, + }, { type: 'bind', source: '/secrets/image_sync_core2026_secret', @@ -42,12 +51,46 @@ const safeRuntime = { pids_limit: 256, }; +const safeBuilder = { + environment: { + NODE_OPTIONS: '--max-old-space-size=3072', + RAYON_NUM_THREADS: '2', + }, + restart: 'unless-stopped', + mem_limit: String(4 * 1024 * 1024 * 1024), + memswap_limit: String(4 * 1024 * 1024 * 1024), + cpus: 4, + pids_limit: 256, +}; + +const safeCaddy = { + restart: 'unless-stopped', + volumes: [ + { + type: 'volume', + source: 'frontend-artifacts', + target: '/srv/frontend-artifacts', + read_only: true, + }, + ], +}; + +const safeServices = () => ({ + runtime: structuredClone(safeRuntime), + builder: structuredClone(safeBuilder), + caddy: structuredClone(safeCaddy), +}); + test('accepts a bounded production runtime', () => { - assert.deepEqual(validateComposeModel({ services: { runtime: safeRuntime } }, 'production'), []); + assert.deepEqual(validateComposeModel({ services: safeServices() }, 'production'), []); }); test('accepts a literal, non-restarting development model', () => { - const services = Object.fromEntries(['postgres', 'redis', 'caddy'].map((name) => [name, { restart: 'no' }])); + const services = safeServices(); + services.postgres = { restart: 'no' }; + services.redis = { restart: 'no' }; + services.builder.restart = 'no'; + services.caddy.restart = 'no'; services.runtime = { ...safeRuntime, restart: 'no', @@ -61,13 +104,19 @@ test('accepts a literal, non-restarting development model', () => { }); test('accepts a bounded, non-restarting production smoke model', () => { - const services = Object.fromEntries(['postgres', 'redis', 'caddy'].map((name) => [name, { restart: 'no' }])); + const services = safeServices(); + services.postgres = { restart: 'no' }; + services.redis = { restart: 'no' }; + services.builder.restart = 'no'; + services.caddy.restart = 'no'; services.runtime = { ...safeRuntime, restart: 'no' }; assert.deepEqual(validateComposeModel({ services }, 'smoke'), []); }); test('rejects an unbounded or production-mode development runtime', () => { - const services = Object.fromEntries(['postgres', 'redis', 'caddy'].map((name) => [name, { restart: 'unless-stopped' }])); + const services = safeServices(); + services.postgres = { restart: 'unless-stopped' }; + services.redis = { restart: 'unless-stopped' }; services.runtime = { environment: { CORE_SOURCE_MODE: 'bind', RUNTIME_MODE: 'production' }, restart: 'unless-stopped', @@ -92,7 +141,9 @@ test('rejects missing or excessive build memory and parallelism limits', () => { RAYON_NUM_THREADS: '8', }, }; - const errors = validateComposeModel({ services: { runtime } }, 'production'); + const services = safeServices(); + services.runtime = runtime; + const errors = validateComposeModel({ services }, 'production'); assert.ok(errors.some((error) => error.includes('Node heap limit'))); assert.ok(errors.some((error) => error.includes('turn daemon Node heap limit'))); assert.ok(errors.some((error) => error.includes('Rayon thread count')));