feat: store editor images in upload bind

This commit is contained in:
2026-08-06 15:52:54 +00:00
parent 9046a131f2
commit dc0e1a6da9
8 changed files with 91 additions and 27 deletions
+6 -3
View File
@@ -136,8 +136,9 @@ service itself is stopped, restore it and run the caller command again.
### Short-lived user-icon uploads ### Short-lived user-icon uploads
Core and Core2026 can store validated account icons in this repository through Core and Core2026 can store validated account icons and editor attachments
`PUT /v1/uploads/user-icons/<client>/<random-32-hex>.<extension>`. Each game through this service with
`PUT /v1/uploads/<user-icons|content>/<client>/<random-32-hex>.<extension>`. Each game
server validates the authenticated user and image first, then sends the raw server validates the authenticated user and image first, then sends the raw
image body with `X-Image-Client`, `X-Image-Expires`, `X-Image-Request-Id`, and image body with `X-Image-Client`, `X-Image-Expires`, `X-Image-Request-Id`, and
`X-Image-Signature` headers. `X-Image-Signature` headers.
@@ -148,7 +149,9 @@ five minutes in the future, so a grant cannot be reused for another path,
content type, body, or later upload. The service also checks the image magic, content type, body, or later upload. The service also checks the image magic,
caller scope, and request replay before it writes one immutable file below the caller scope, and request replay before it writes one immutable file below the
host bind directory `runtime-data/uploads`. User uploads are deliberately not host bind directory `runtime-data/uploads`. User uploads are deliberately not
added to Git; Nginx exposes that bind read-only at `/icons/users/`. added to Git; Nginx exposes that bind read-only at `/icons/users/` for account
icons and `/uploads/` for editor content. User icons retain the 50KB limit;
editor content retains the existing 1MB limit.
Create separate upload secrets with `deploy/scripts/init-secrets.sh`. Mount only Create separate upload secrets with `deploy/scripts/init-secrets.sh`. Mount only
the matching `image_upload_core_secret` or `image_upload_core2026_secret` on the the matching `image_upload_core_secret` or `image_upload_core2026_secret` on the
+2 -1
View File
@@ -25,6 +25,7 @@ services:
IMAGE_SYNC_CLIENT_SECRET_FILES: core=/run/secrets/image_sync_core_secret,core2026=/run/secrets/image_sync_core2026_secret IMAGE_SYNC_CLIENT_SECRET_FILES: core=/run/secrets/image_sync_core_secret,core2026=/run/secrets/image_sync_core2026_secret
IMAGE_UPLOAD_CLIENT_SECRET_FILES: core=/run/secrets/image_upload_core_secret,core2026=/run/secrets/image_upload_core2026_secret IMAGE_UPLOAD_CLIENT_SECRET_FILES: core=/run/secrets/image_upload_core_secret,core2026=/run/secrets/image_upload_core2026_secret
MAX_UPLOAD_BYTES: "51200" MAX_UPLOAD_BYTES: "51200"
MAX_CONTENT_UPLOAD_BYTES: "1048576"
volumes: volumes:
- type: bind - type: bind
source: ${IMAGE_REPOSITORY_PATH:-.} source: ${IMAGE_REPOSITORY_PATH:-.}
@@ -73,7 +74,7 @@ services:
read_only: true read_only: true
- type: bind - type: bind
source: ./runtime-data/uploads source: ./runtime-data/uploads
target: /srv/user-icons target: /srv/uploads
read_only: true read_only: true
tmpfs: tmpfs:
- /tmp:size=8m,mode=1777 - /tmp:size=8m,mode=1777
+18 -5
View File
@@ -69,9 +69,9 @@ http {
proxy_request_buffering on; proxy_request_buffering on;
} }
location ^~ /v1/uploads/user-icons/ { location ^~ /v1/uploads/ {
limit_except PUT { deny all; } limit_except PUT { deny all; }
client_max_body_size 50k; client_max_body_size 1m;
proxy_pass http://image-hook:8081; proxy_pass http://image-hook:8081;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Image-Client $http_x_image_client; proxy_set_header X-Image-Client $http_x_image_client;
@@ -92,8 +92,8 @@ http {
add_header X-Content-Type-Options nosniff always; add_header X-Content-Type-Options nosniff always;
} }
location ^~ /icons/users/ { location ~ "^/icons/users/([a-z0-9][a-z0-9_-]{1,31})/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$" {
alias /srv/user-icons/; alias /srv/uploads/user-icons/$1/$2;
etag on; etag on;
expires 1y; expires 1y;
add_header Cache-Control "public, immutable" always; add_header Cache-Control "public, immutable" always;
@@ -101,7 +101,20 @@ http {
add_header X-Content-Type-Options nosniff always; add_header X-Content-Type-Options nosniff always;
} }
location ^~ /icons/ { location /icons/users/ { return 404; }
location ~ "^/uploads/([a-z0-9][a-z0-9_-]{1,31})/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$" {
alias /srv/uploads/content/$1/$2;
etag on;
expires 1y;
add_header Cache-Control "public, immutable" always;
add_header Access-Control-Allow-Origin "*" always;
add_header X-Content-Type-Options nosniff always;
}
location /uploads/ { return 404; }
location /icons/ {
try_files $uri =404; try_files $uri =404;
add_header Access-Control-Allow-Origin "*" always; add_header Access-Control-Allow-Origin "*" always;
add_header X-Content-Type-Options nosniff always; add_header X-Content-Type-Options nosniff always;
+1
View File
@@ -77,6 +77,7 @@ export function loadConfig() {
uploadClientSecrets: clientSecrets('IMAGE_UPLOAD_CLIENT_SECRET_FILES'), uploadClientSecrets: clientSecrets('IMAGE_UPLOAD_CLIENT_SECRET_FILES'),
maxBodyBytes: Number(text('MAX_BODY_BYTES', '1048576')), maxBodyBytes: Number(text('MAX_BODY_BYTES', '1048576')),
maxUploadBytes: Number(text('MAX_UPLOAD_BYTES', '51200')), maxUploadBytes: Number(text('MAX_UPLOAD_BYTES', '51200')),
maxContentUploadBytes: Number(text('MAX_CONTENT_UPLOAD_BYTES', '1048576')),
uploadRoot: text('IMAGE_UPLOAD_ROOT', '/var/lib/image-hook/uploads'), uploadRoot: text('IMAGE_UPLOAD_ROOT', '/var/lib/image-hook/uploads'),
uploadStatePath: text('IMAGE_UPLOAD_STATE_PATH', '/var/lib/image-hook/upload-state.json'), uploadStatePath: text('IMAGE_UPLOAD_STATE_PATH', '/var/lib/image-hook/upload-state.json'),
}; };
+13 -9
View File
@@ -38,7 +38,7 @@ function parseJson(body) {
function hasImageSignature(body, extension) { function hasImageSignature(body, extension) {
if (extension === 'png') return body.length >= 8 && body.subarray(0, 8).equals(Buffer.from('89504e470d0a1a0a', 'hex')); if (extension === 'png') return body.length >= 8 && body.subarray(0, 8).equals(Buffer.from('89504e470d0a1a0a', 'hex'));
if (extension === 'jpg') return body.length >= 3 && body[0] === 0xff && body[1] === 0xd8 && body[2] === 0xff; if (extension === 'jpg' || extension === 'jpeg') return body.length >= 3 && body[0] === 0xff && body[1] === 0xd8 && body[2] === 0xff;
if (extension === 'gif') return body.length >= 6 && ['GIF87a', 'GIF89a'].includes(body.subarray(0, 6).toString('ascii')); if (extension === 'gif') return body.length >= 6 && ['GIF87a', 'GIF89a'].includes(body.subarray(0, 6).toString('ascii'));
if (extension === 'webp') return body.length >= 12 && body.subarray(0, 4).toString('ascii') === 'RIFF' if (extension === 'webp') return body.length >= 12 && body.subarray(0, 4).toString('ascii') === 'RIFF'
&& body.subarray(8, 12).toString('ascii') === 'WEBP'; && body.subarray(8, 12).toString('ascii') === 'WEBP';
@@ -149,23 +149,26 @@ export async function createApp(config = loadConfig(), dependencies = {}) {
}); });
return json(response, 200, { ok: true, ...result }); return json(response, 200, { ok: true, ...result });
} }
if (request.method === 'PUT' && url.pathname.startsWith('/v1/uploads/user-icons/')) { if (request.method === 'PUT' && url.pathname.startsWith('/v1/uploads/')) {
const client = request.headers['x-image-client']; const client = request.headers['x-image-client'];
const expires = request.headers['x-image-expires']; const expires = request.headers['x-image-expires'];
const requestId = request.headers['x-image-request-id']; const requestId = request.headers['x-image-request-id'];
const contentType = request.headers['content-type']?.toLowerCase() ?? ''; const contentType = request.headers['content-type']?.toLowerCase() ?? '';
const knownClient = typeof client === 'string' && Object.hasOwn(config.uploadClientSecrets, client); const knownClient = typeof client === 'string' && Object.hasOwn(config.uploadClientSecrets, client);
const match = url.pathname.match(/^\/v1\/uploads\/user-icons\/([a-z0-9][a-z0-9_-]{1,31})\/([a-f0-9]{32})\.(avif|webp|jpg|png|gif)$/); const match = url.pathname.match(/^\/v1\/uploads\/(user-icons|content)\/([a-z0-9][a-z0-9_-]{1,31})\/([a-f0-9]{32})\.(avif|webp|jpe?g|png|gif)$/);
if (!match || match[1] !== client) { if (!match || match[2] !== client) {
throw new DeploymentError('Invalid upload path', 400); throw new DeploymentError('Invalid upload path', 400);
} }
const mimeByExtension = { const mimeByExtension = {
avif: 'image/avif', webp: 'image/webp', jpg: 'image/jpeg', png: 'image/png', gif: 'image/gif', avif: 'image/avif', webp: 'image/webp', jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif',
}; };
if (contentType !== mimeByExtension[match[3]]) { if (contentType !== mimeByExtension[match[4]]) {
throw new DeploymentError('Content-Type does not match upload path', 415); throw new DeploymentError('Content-Type does not match upload path', 415);
} }
const body = await readBody(request, config.maxUploadBytes); const body = await readBody(
request,
match[1] === 'user-icons' ? config.maxUploadBytes : config.maxContentUploadBytes,
);
const signatureValid = verifyUploadSignature({ const signatureValid = verifyUploadSignature({
secret: knownClient ? config.uploadClientSecrets[client] : 'invalid-client-secret'.padEnd(32, '!'), secret: knownClient ? config.uploadClientSecrets[client] : 'invalid-client-secret'.padEnd(32, '!'),
expires, expires,
@@ -178,13 +181,14 @@ export async function createApp(config = loadConfig(), dependencies = {}) {
if (!knownClient || !signatureValid) { if (!knownClient || !signatureValid) {
return json(response, 401, { ok: false, reason: 'invalid or expired upload grant' }); return json(response, 401, { ok: false, reason: 'invalid or expired upload grant' });
} }
if (!hasImageSignature(body, match[3])) { if (!hasImageSignature(body, match[4])) {
throw new DeploymentError('Body is not the declared image format', 400); throw new DeploymentError('Body is not the declared image format', 400);
} }
const result = await uploadStore.store({ const result = await uploadStore.store({
requestKey: `${client}:${requestId}`, requestKey: `${client}:${requestId}`,
category: match[1],
client, client,
filename: `${match[2]}.${match[3]}`, filename: `${match[3]}.${match[4]}`,
body, body,
}); });
const urls = config.publicBases.map((base) => `${base}/${result.path}`); const urls = config.publicBases.map((base) => `${base}/${result.path}`);
+8 -5
View File
@@ -22,14 +22,17 @@ export class UploadStore {
} }
} }
store({ requestKey, client, filename, body }) { store({ requestKey, category, client, filename, body }) {
const operation = this.queue.then(async () => { const operation = this.queue.then(async () => {
if (!/^[a-z0-9][a-z0-9_-]{1,31}$/.test(client) if (!['user-icons', 'content'].includes(category)
|| !/^[a-f0-9]{32}\.(?:avif|webp|jpg|png|gif)$/.test(filename)) { || !/^[a-z0-9][a-z0-9_-]{1,31}$/.test(client)
|| !/^[a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif)$/.test(filename)) {
throw new DeploymentError('Invalid upload path', 400); throw new DeploymentError('Invalid upload path', 400);
} }
const relativePath = `${client}/${filename}`; const relativePath = `${category}/${client}/${filename}`;
const path = `icons/users/${relativePath}`; const path = category === 'user-icons'
? `icons/users/${client}/${filename}`
: `uploads/${client}/${filename}`;
const digest = createHash('sha256').update(body).digest('hex'); const digest = createHash('sha256').update(body).digest('hex');
const previous = this.uploads.find((upload) => upload.key === requestKey); const previous = this.uploads.find((upload) => upload.key === requestKey);
if (previous) { if (previous) {
+27
View File
@@ -61,6 +61,7 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay
const config = { const config = {
maxBodyBytes: 4096, maxBodyBytes: 4096,
maxUploadBytes: 51200, maxUploadBytes: 51200,
maxContentUploadBytes: 1048576,
syncClientSecrets: { core: 's'.repeat(32) }, syncClientSecrets: { core: 's'.repeat(32) },
uploadClientSecrets: { core2026: secret }, uploadClientSecrets: { core2026: secret },
publicBases: ['https://sam-image.hided.net', 'https://sam.hided.net/image'], publicBases: ['https://sam-image.hided.net', 'https://sam.hided.net/image'],
@@ -86,6 +87,7 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay
assert.equal(accepted.status, 201); assert.equal(accepted.status, 201);
assert.deepEqual(calls[0], { assert.deepEqual(calls[0], {
requestKey: `core2026:${requestId}`, requestKey: `core2026:${requestId}`,
category: 'user-icons',
client: 'core2026', client: 'core2026',
filename: `${'a'.repeat(32)}.png`, filename: `${'a'.repeat(32)}.png`,
body, body,
@@ -95,6 +97,31 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay
`https://sam.hided.net/image/icons/users/core2026/${'a'.repeat(32)}.png`, `https://sam.hided.net/image/icons/users/core2026/${'a'.repeat(32)}.png`,
]); ]);
const contentPath = `/v1/uploads/content/core2026/${'b'.repeat(32)}.webp`;
const contentBody = Buffer.concat([Buffer.from('RIFF'), Buffer.alloc(4), Buffer.from('WEBP'), Buffer.alloc(4)]);
const contentRequestId = 'content-request-1234';
const contentSignature = uploadSignature(secret, {
expires,
requestId: contentRequestId,
pathname: contentPath,
contentType: 'image/webp',
body: contentBody,
});
const contentResponse = await fetch(`http://127.0.0.1:${address.port}${contentPath}`, {
method: 'PUT',
headers: {
'content-type': 'image/webp',
'x-image-client': 'core2026',
'x-image-expires': expires,
'x-image-request-id': contentRequestId,
'x-image-signature': contentSignature,
},
body: contentBody,
});
assert.equal(contentResponse.status, 201);
assert.equal(calls[1].category, 'content');
assert.equal(calls[1].filename, `${'b'.repeat(32)}.webp`);
const tampered = await fetch(`http://127.0.0.1:${address.port}${pathname}`, { const tampered = await fetch(`http://127.0.0.1:${address.port}${pathname}`, {
method: 'PUT', headers, body: Buffer.from('89504e470d0a1a0affffffff', 'hex'), method: 'PUT', headers, body: Buffer.from('89504e470d0a1a0affffffff', 'hex'),
}); });
+16 -4
View File
@@ -13,19 +13,20 @@ test('stores uploads only in the bind directory and persists replay state', asyn
const filename = `${'a'.repeat(32)}.png`; const filename = `${'a'.repeat(32)}.png`;
const store = new UploadStore(config); const store = new UploadStore(config);
await store.initialize(); await store.initialize();
const first = await store.store({ requestKey: 'core2026:request-1', client: 'core2026', filename, body }); const first = await store.store({ requestKey: 'core2026:request-1', category: 'user-icons', client: 'core2026', filename, body });
assert.deepEqual(first, { duplicate: false, path: `icons/users/core2026/${filename}` }); assert.deepEqual(first, { duplicate: false, path: `icons/users/core2026/${filename}` });
assert.equal(await readFile(join(root, 'uploads', 'core2026', filename), 'utf8'), 'immutable image bytes'); assert.equal(await readFile(join(root, 'uploads', 'user-icons', 'core2026', filename), 'utf8'), 'immutable image bytes');
const restarted = new UploadStore(config); const restarted = new UploadStore(config);
await restarted.initialize(); await restarted.initialize();
assert.deepEqual( assert.deepEqual(
await restarted.store({ requestKey: 'core2026:request-1', client: 'core2026', filename, body }), await restarted.store({ requestKey: 'core2026:request-1', category: 'user-icons', client: 'core2026', filename, body }),
{ duplicate: true, path: `icons/users/core2026/${filename}` }, { duplicate: true, path: `icons/users/core2026/${filename}` },
); );
await assert.rejects( await assert.rejects(
restarted.store({ restarted.store({
requestKey: 'core2026:request-1', requestKey: 'core2026:request-1',
category: 'user-icons',
client: 'core2026', client: 'core2026',
filename: `${'b'.repeat(32)}.png`, filename: `${'b'.repeat(32)}.png`,
body, body,
@@ -33,7 +34,18 @@ test('stores uploads only in the bind directory and persists replay state', asyn
/already used/, /already used/,
); );
await assert.rejects( await assert.rejects(
restarted.store({ requestKey: 'core2026:request-2', client: '../escape', filename, body }), restarted.store({ requestKey: 'core2026:request-2', category: 'user-icons', client: '../escape', filename, body }),
/Invalid upload path/, /Invalid upload path/,
); );
const contentFilename = `${'c'.repeat(32)}.webp`;
assert.deepEqual(
await restarted.store({
requestKey: 'core2026:content-1',
category: 'content',
client: 'core2026',
filename: contentFilename,
body: Buffer.from('content image'),
}),
{ duplicate: false, path: `uploads/core2026/${contentFilename}` },
);
}); });