feat: store editor images in upload bind
This commit is contained in:
@@ -136,8 +136,9 @@ service itself is stopped, restore it and run the caller command again.
|
|||||||
|
|
||||||
### Short-lived user-icon uploads
|
### Short-lived user-icon uploads
|
||||||
|
|
||||||
Core and Core2026 can store validated account icons in this repository through
|
Core and Core2026 can store validated account icons and editor attachments
|
||||||
`PUT /v1/uploads/user-icons/<client>/<random-32-hex>.<extension>`. Each game
|
through this service with
|
||||||
|
`PUT /v1/uploads/<user-icons|content>/<client>/<random-32-hex>.<extension>`. Each game
|
||||||
server validates the authenticated user and image first, then sends the raw
|
server validates the authenticated user and image first, then sends the raw
|
||||||
image body with `X-Image-Client`, `X-Image-Expires`, `X-Image-Request-Id`, and
|
image body with `X-Image-Client`, `X-Image-Expires`, `X-Image-Request-Id`, and
|
||||||
`X-Image-Signature` headers.
|
`X-Image-Signature` headers.
|
||||||
@@ -148,7 +149,9 @@ five minutes in the future, so a grant cannot be reused for another path,
|
|||||||
content type, body, or later upload. The service also checks the image magic,
|
content type, body, or later upload. The service also checks the image magic,
|
||||||
caller scope, and request replay before it writes one immutable file below the
|
caller scope, and request replay before it writes one immutable file below the
|
||||||
host bind directory `runtime-data/uploads`. User uploads are deliberately not
|
host bind directory `runtime-data/uploads`. User uploads are deliberately not
|
||||||
added to Git; Nginx exposes that bind read-only at `/icons/users/`.
|
added to Git; Nginx exposes that bind read-only at `/icons/users/` for account
|
||||||
|
icons and `/uploads/` for editor content. User icons retain the 50KB limit;
|
||||||
|
editor content retains the existing 1MB limit.
|
||||||
|
|
||||||
Create separate upload secrets with `deploy/scripts/init-secrets.sh`. Mount only
|
Create separate upload secrets with `deploy/scripts/init-secrets.sh`. Mount only
|
||||||
the matching `image_upload_core_secret` or `image_upload_core2026_secret` on the
|
the matching `image_upload_core_secret` or `image_upload_core2026_secret` on the
|
||||||
|
|||||||
+2
-1
@@ -25,6 +25,7 @@ services:
|
|||||||
IMAGE_SYNC_CLIENT_SECRET_FILES: core=/run/secrets/image_sync_core_secret,core2026=/run/secrets/image_sync_core2026_secret
|
IMAGE_SYNC_CLIENT_SECRET_FILES: core=/run/secrets/image_sync_core_secret,core2026=/run/secrets/image_sync_core2026_secret
|
||||||
IMAGE_UPLOAD_CLIENT_SECRET_FILES: core=/run/secrets/image_upload_core_secret,core2026=/run/secrets/image_upload_core2026_secret
|
IMAGE_UPLOAD_CLIENT_SECRET_FILES: core=/run/secrets/image_upload_core_secret,core2026=/run/secrets/image_upload_core2026_secret
|
||||||
MAX_UPLOAD_BYTES: "51200"
|
MAX_UPLOAD_BYTES: "51200"
|
||||||
|
MAX_CONTENT_UPLOAD_BYTES: "1048576"
|
||||||
volumes:
|
volumes:
|
||||||
- type: bind
|
- type: bind
|
||||||
source: ${IMAGE_REPOSITORY_PATH:-.}
|
source: ${IMAGE_REPOSITORY_PATH:-.}
|
||||||
@@ -73,7 +74,7 @@ services:
|
|||||||
read_only: true
|
read_only: true
|
||||||
- type: bind
|
- type: bind
|
||||||
source: ./runtime-data/uploads
|
source: ./runtime-data/uploads
|
||||||
target: /srv/user-icons
|
target: /srv/uploads
|
||||||
read_only: true
|
read_only: true
|
||||||
tmpfs:
|
tmpfs:
|
||||||
- /tmp:size=8m,mode=1777
|
- /tmp:size=8m,mode=1777
|
||||||
|
|||||||
@@ -69,9 +69,9 @@ http {
|
|||||||
proxy_request_buffering on;
|
proxy_request_buffering on;
|
||||||
}
|
}
|
||||||
|
|
||||||
location ^~ /v1/uploads/user-icons/ {
|
location ^~ /v1/uploads/ {
|
||||||
limit_except PUT { deny all; }
|
limit_except PUT { deny all; }
|
||||||
client_max_body_size 50k;
|
client_max_body_size 1m;
|
||||||
proxy_pass http://image-hook:8081;
|
proxy_pass http://image-hook:8081;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Image-Client $http_x_image_client;
|
proxy_set_header X-Image-Client $http_x_image_client;
|
||||||
@@ -92,8 +92,8 @@ http {
|
|||||||
add_header X-Content-Type-Options nosniff always;
|
add_header X-Content-Type-Options nosniff always;
|
||||||
}
|
}
|
||||||
|
|
||||||
location ^~ /icons/users/ {
|
location ~ "^/icons/users/([a-z0-9][a-z0-9_-]{1,31})/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$" {
|
||||||
alias /srv/user-icons/;
|
alias /srv/uploads/user-icons/$1/$2;
|
||||||
etag on;
|
etag on;
|
||||||
expires 1y;
|
expires 1y;
|
||||||
add_header Cache-Control "public, immutable" always;
|
add_header Cache-Control "public, immutable" always;
|
||||||
@@ -101,7 +101,20 @@ http {
|
|||||||
add_header X-Content-Type-Options nosniff always;
|
add_header X-Content-Type-Options nosniff always;
|
||||||
}
|
}
|
||||||
|
|
||||||
location ^~ /icons/ {
|
location /icons/users/ { return 404; }
|
||||||
|
|
||||||
|
location ~ "^/uploads/([a-z0-9][a-z0-9_-]{1,31})/([a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif))$" {
|
||||||
|
alias /srv/uploads/content/$1/$2;
|
||||||
|
etag on;
|
||||||
|
expires 1y;
|
||||||
|
add_header Cache-Control "public, immutable" always;
|
||||||
|
add_header Access-Control-Allow-Origin "*" always;
|
||||||
|
add_header X-Content-Type-Options nosniff always;
|
||||||
|
}
|
||||||
|
|
||||||
|
location /uploads/ { return 404; }
|
||||||
|
|
||||||
|
location /icons/ {
|
||||||
try_files $uri =404;
|
try_files $uri =404;
|
||||||
add_header Access-Control-Allow-Origin "*" always;
|
add_header Access-Control-Allow-Origin "*" always;
|
||||||
add_header X-Content-Type-Options nosniff always;
|
add_header X-Content-Type-Options nosniff always;
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ export function loadConfig() {
|
|||||||
uploadClientSecrets: clientSecrets('IMAGE_UPLOAD_CLIENT_SECRET_FILES'),
|
uploadClientSecrets: clientSecrets('IMAGE_UPLOAD_CLIENT_SECRET_FILES'),
|
||||||
maxBodyBytes: Number(text('MAX_BODY_BYTES', '1048576')),
|
maxBodyBytes: Number(text('MAX_BODY_BYTES', '1048576')),
|
||||||
maxUploadBytes: Number(text('MAX_UPLOAD_BYTES', '51200')),
|
maxUploadBytes: Number(text('MAX_UPLOAD_BYTES', '51200')),
|
||||||
|
maxContentUploadBytes: Number(text('MAX_CONTENT_UPLOAD_BYTES', '1048576')),
|
||||||
uploadRoot: text('IMAGE_UPLOAD_ROOT', '/var/lib/image-hook/uploads'),
|
uploadRoot: text('IMAGE_UPLOAD_ROOT', '/var/lib/image-hook/uploads'),
|
||||||
uploadStatePath: text('IMAGE_UPLOAD_STATE_PATH', '/var/lib/image-hook/upload-state.json'),
|
uploadStatePath: text('IMAGE_UPLOAD_STATE_PATH', '/var/lib/image-hook/upload-state.json'),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ function parseJson(body) {
|
|||||||
|
|
||||||
function hasImageSignature(body, extension) {
|
function hasImageSignature(body, extension) {
|
||||||
if (extension === 'png') return body.length >= 8 && body.subarray(0, 8).equals(Buffer.from('89504e470d0a1a0a', 'hex'));
|
if (extension === 'png') return body.length >= 8 && body.subarray(0, 8).equals(Buffer.from('89504e470d0a1a0a', 'hex'));
|
||||||
if (extension === 'jpg') return body.length >= 3 && body[0] === 0xff && body[1] === 0xd8 && body[2] === 0xff;
|
if (extension === 'jpg' || extension === 'jpeg') return body.length >= 3 && body[0] === 0xff && body[1] === 0xd8 && body[2] === 0xff;
|
||||||
if (extension === 'gif') return body.length >= 6 && ['GIF87a', 'GIF89a'].includes(body.subarray(0, 6).toString('ascii'));
|
if (extension === 'gif') return body.length >= 6 && ['GIF87a', 'GIF89a'].includes(body.subarray(0, 6).toString('ascii'));
|
||||||
if (extension === 'webp') return body.length >= 12 && body.subarray(0, 4).toString('ascii') === 'RIFF'
|
if (extension === 'webp') return body.length >= 12 && body.subarray(0, 4).toString('ascii') === 'RIFF'
|
||||||
&& body.subarray(8, 12).toString('ascii') === 'WEBP';
|
&& body.subarray(8, 12).toString('ascii') === 'WEBP';
|
||||||
@@ -149,23 +149,26 @@ export async function createApp(config = loadConfig(), dependencies = {}) {
|
|||||||
});
|
});
|
||||||
return json(response, 200, { ok: true, ...result });
|
return json(response, 200, { ok: true, ...result });
|
||||||
}
|
}
|
||||||
if (request.method === 'PUT' && url.pathname.startsWith('/v1/uploads/user-icons/')) {
|
if (request.method === 'PUT' && url.pathname.startsWith('/v1/uploads/')) {
|
||||||
const client = request.headers['x-image-client'];
|
const client = request.headers['x-image-client'];
|
||||||
const expires = request.headers['x-image-expires'];
|
const expires = request.headers['x-image-expires'];
|
||||||
const requestId = request.headers['x-image-request-id'];
|
const requestId = request.headers['x-image-request-id'];
|
||||||
const contentType = request.headers['content-type']?.toLowerCase() ?? '';
|
const contentType = request.headers['content-type']?.toLowerCase() ?? '';
|
||||||
const knownClient = typeof client === 'string' && Object.hasOwn(config.uploadClientSecrets, client);
|
const knownClient = typeof client === 'string' && Object.hasOwn(config.uploadClientSecrets, client);
|
||||||
const match = url.pathname.match(/^\/v1\/uploads\/user-icons\/([a-z0-9][a-z0-9_-]{1,31})\/([a-f0-9]{32})\.(avif|webp|jpg|png|gif)$/);
|
const match = url.pathname.match(/^\/v1\/uploads\/(user-icons|content)\/([a-z0-9][a-z0-9_-]{1,31})\/([a-f0-9]{32})\.(avif|webp|jpe?g|png|gif)$/);
|
||||||
if (!match || match[1] !== client) {
|
if (!match || match[2] !== client) {
|
||||||
throw new DeploymentError('Invalid upload path', 400);
|
throw new DeploymentError('Invalid upload path', 400);
|
||||||
}
|
}
|
||||||
const mimeByExtension = {
|
const mimeByExtension = {
|
||||||
avif: 'image/avif', webp: 'image/webp', jpg: 'image/jpeg', png: 'image/png', gif: 'image/gif',
|
avif: 'image/avif', webp: 'image/webp', jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif',
|
||||||
};
|
};
|
||||||
if (contentType !== mimeByExtension[match[3]]) {
|
if (contentType !== mimeByExtension[match[4]]) {
|
||||||
throw new DeploymentError('Content-Type does not match upload path', 415);
|
throw new DeploymentError('Content-Type does not match upload path', 415);
|
||||||
}
|
}
|
||||||
const body = await readBody(request, config.maxUploadBytes);
|
const body = await readBody(
|
||||||
|
request,
|
||||||
|
match[1] === 'user-icons' ? config.maxUploadBytes : config.maxContentUploadBytes,
|
||||||
|
);
|
||||||
const signatureValid = verifyUploadSignature({
|
const signatureValid = verifyUploadSignature({
|
||||||
secret: knownClient ? config.uploadClientSecrets[client] : 'invalid-client-secret'.padEnd(32, '!'),
|
secret: knownClient ? config.uploadClientSecrets[client] : 'invalid-client-secret'.padEnd(32, '!'),
|
||||||
expires,
|
expires,
|
||||||
@@ -178,13 +181,14 @@ export async function createApp(config = loadConfig(), dependencies = {}) {
|
|||||||
if (!knownClient || !signatureValid) {
|
if (!knownClient || !signatureValid) {
|
||||||
return json(response, 401, { ok: false, reason: 'invalid or expired upload grant' });
|
return json(response, 401, { ok: false, reason: 'invalid or expired upload grant' });
|
||||||
}
|
}
|
||||||
if (!hasImageSignature(body, match[3])) {
|
if (!hasImageSignature(body, match[4])) {
|
||||||
throw new DeploymentError('Body is not the declared image format', 400);
|
throw new DeploymentError('Body is not the declared image format', 400);
|
||||||
}
|
}
|
||||||
const result = await uploadStore.store({
|
const result = await uploadStore.store({
|
||||||
requestKey: `${client}:${requestId}`,
|
requestKey: `${client}:${requestId}`,
|
||||||
|
category: match[1],
|
||||||
client,
|
client,
|
||||||
filename: `${match[2]}.${match[3]}`,
|
filename: `${match[3]}.${match[4]}`,
|
||||||
body,
|
body,
|
||||||
});
|
});
|
||||||
const urls = config.publicBases.map((base) => `${base}/${result.path}`);
|
const urls = config.publicBases.map((base) => `${base}/${result.path}`);
|
||||||
|
|||||||
@@ -22,14 +22,17 @@ export class UploadStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
store({ requestKey, client, filename, body }) {
|
store({ requestKey, category, client, filename, body }) {
|
||||||
const operation = this.queue.then(async () => {
|
const operation = this.queue.then(async () => {
|
||||||
if (!/^[a-z0-9][a-z0-9_-]{1,31}$/.test(client)
|
if (!['user-icons', 'content'].includes(category)
|
||||||
|| !/^[a-f0-9]{32}\.(?:avif|webp|jpg|png|gif)$/.test(filename)) {
|
|| !/^[a-z0-9][a-z0-9_-]{1,31}$/.test(client)
|
||||||
|
|| !/^[a-f0-9]{32}\.(?:avif|webp|jpe?g|png|gif)$/.test(filename)) {
|
||||||
throw new DeploymentError('Invalid upload path', 400);
|
throw new DeploymentError('Invalid upload path', 400);
|
||||||
}
|
}
|
||||||
const relativePath = `${client}/${filename}`;
|
const relativePath = `${category}/${client}/${filename}`;
|
||||||
const path = `icons/users/${relativePath}`;
|
const path = category === 'user-icons'
|
||||||
|
? `icons/users/${client}/${filename}`
|
||||||
|
: `uploads/${client}/${filename}`;
|
||||||
const digest = createHash('sha256').update(body).digest('hex');
|
const digest = createHash('sha256').update(body).digest('hex');
|
||||||
const previous = this.uploads.find((upload) => upload.key === requestKey);
|
const previous = this.uploads.find((upload) => upload.key === requestKey);
|
||||||
if (previous) {
|
if (previous) {
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay
|
|||||||
const config = {
|
const config = {
|
||||||
maxBodyBytes: 4096,
|
maxBodyBytes: 4096,
|
||||||
maxUploadBytes: 51200,
|
maxUploadBytes: 51200,
|
||||||
|
maxContentUploadBytes: 1048576,
|
||||||
syncClientSecrets: { core: 's'.repeat(32) },
|
syncClientSecrets: { core: 's'.repeat(32) },
|
||||||
uploadClientSecrets: { core2026: secret },
|
uploadClientSecrets: { core2026: secret },
|
||||||
publicBases: ['https://sam-image.hided.net', 'https://sam.hided.net/image'],
|
publicBases: ['https://sam-image.hided.net', 'https://sam.hided.net/image'],
|
||||||
@@ -86,6 +87,7 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay
|
|||||||
assert.equal(accepted.status, 201);
|
assert.equal(accepted.status, 201);
|
||||||
assert.deepEqual(calls[0], {
|
assert.deepEqual(calls[0], {
|
||||||
requestKey: `core2026:${requestId}`,
|
requestKey: `core2026:${requestId}`,
|
||||||
|
category: 'user-icons',
|
||||||
client: 'core2026',
|
client: 'core2026',
|
||||||
filename: `${'a'.repeat(32)}.png`,
|
filename: `${'a'.repeat(32)}.png`,
|
||||||
body,
|
body,
|
||||||
@@ -95,6 +97,31 @@ test('upload endpoint accepts a short-lived body-bound grant and rejects replay
|
|||||||
`https://sam.hided.net/image/icons/users/core2026/${'a'.repeat(32)}.png`,
|
`https://sam.hided.net/image/icons/users/core2026/${'a'.repeat(32)}.png`,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
const contentPath = `/v1/uploads/content/core2026/${'b'.repeat(32)}.webp`;
|
||||||
|
const contentBody = Buffer.concat([Buffer.from('RIFF'), Buffer.alloc(4), Buffer.from('WEBP'), Buffer.alloc(4)]);
|
||||||
|
const contentRequestId = 'content-request-1234';
|
||||||
|
const contentSignature = uploadSignature(secret, {
|
||||||
|
expires,
|
||||||
|
requestId: contentRequestId,
|
||||||
|
pathname: contentPath,
|
||||||
|
contentType: 'image/webp',
|
||||||
|
body: contentBody,
|
||||||
|
});
|
||||||
|
const contentResponse = await fetch(`http://127.0.0.1:${address.port}${contentPath}`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: {
|
||||||
|
'content-type': 'image/webp',
|
||||||
|
'x-image-client': 'core2026',
|
||||||
|
'x-image-expires': expires,
|
||||||
|
'x-image-request-id': contentRequestId,
|
||||||
|
'x-image-signature': contentSignature,
|
||||||
|
},
|
||||||
|
body: contentBody,
|
||||||
|
});
|
||||||
|
assert.equal(contentResponse.status, 201);
|
||||||
|
assert.equal(calls[1].category, 'content');
|
||||||
|
assert.equal(calls[1].filename, `${'b'.repeat(32)}.webp`);
|
||||||
|
|
||||||
const tampered = await fetch(`http://127.0.0.1:${address.port}${pathname}`, {
|
const tampered = await fetch(`http://127.0.0.1:${address.port}${pathname}`, {
|
||||||
method: 'PUT', headers, body: Buffer.from('89504e470d0a1a0affffffff', 'hex'),
|
method: 'PUT', headers, body: Buffer.from('89504e470d0a1a0affffffff', 'hex'),
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -13,19 +13,20 @@ test('stores uploads only in the bind directory and persists replay state', asyn
|
|||||||
const filename = `${'a'.repeat(32)}.png`;
|
const filename = `${'a'.repeat(32)}.png`;
|
||||||
const store = new UploadStore(config);
|
const store = new UploadStore(config);
|
||||||
await store.initialize();
|
await store.initialize();
|
||||||
const first = await store.store({ requestKey: 'core2026:request-1', client: 'core2026', filename, body });
|
const first = await store.store({ requestKey: 'core2026:request-1', category: 'user-icons', client: 'core2026', filename, body });
|
||||||
assert.deepEqual(first, { duplicate: false, path: `icons/users/core2026/${filename}` });
|
assert.deepEqual(first, { duplicate: false, path: `icons/users/core2026/${filename}` });
|
||||||
assert.equal(await readFile(join(root, 'uploads', 'core2026', filename), 'utf8'), 'immutable image bytes');
|
assert.equal(await readFile(join(root, 'uploads', 'user-icons', 'core2026', filename), 'utf8'), 'immutable image bytes');
|
||||||
|
|
||||||
const restarted = new UploadStore(config);
|
const restarted = new UploadStore(config);
|
||||||
await restarted.initialize();
|
await restarted.initialize();
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
await restarted.store({ requestKey: 'core2026:request-1', client: 'core2026', filename, body }),
|
await restarted.store({ requestKey: 'core2026:request-1', category: 'user-icons', client: 'core2026', filename, body }),
|
||||||
{ duplicate: true, path: `icons/users/core2026/${filename}` },
|
{ duplicate: true, path: `icons/users/core2026/${filename}` },
|
||||||
);
|
);
|
||||||
await assert.rejects(
|
await assert.rejects(
|
||||||
restarted.store({
|
restarted.store({
|
||||||
requestKey: 'core2026:request-1',
|
requestKey: 'core2026:request-1',
|
||||||
|
category: 'user-icons',
|
||||||
client: 'core2026',
|
client: 'core2026',
|
||||||
filename: `${'b'.repeat(32)}.png`,
|
filename: `${'b'.repeat(32)}.png`,
|
||||||
body,
|
body,
|
||||||
@@ -33,7 +34,18 @@ test('stores uploads only in the bind directory and persists replay state', asyn
|
|||||||
/already used/,
|
/already used/,
|
||||||
);
|
);
|
||||||
await assert.rejects(
|
await assert.rejects(
|
||||||
restarted.store({ requestKey: 'core2026:request-2', client: '../escape', filename, body }),
|
restarted.store({ requestKey: 'core2026:request-2', category: 'user-icons', client: '../escape', filename, body }),
|
||||||
/Invalid upload path/,
|
/Invalid upload path/,
|
||||||
);
|
);
|
||||||
|
const contentFilename = `${'c'.repeat(32)}.webp`;
|
||||||
|
assert.deepEqual(
|
||||||
|
await restarted.store({
|
||||||
|
requestKey: 'core2026:content-1',
|
||||||
|
category: 'content',
|
||||||
|
client: 'core2026',
|
||||||
|
filename: contentFilename,
|
||||||
|
body: Buffer.from('content image'),
|
||||||
|
}),
|
||||||
|
{ duplicate: false, path: `uploads/core2026/${contentFilename}` },
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user