cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Instant upload: register already-owned content by hash, zero bytes on the wire

Browse Source 
Phase 0 of the delta-sync plan. Re-uploading a file the user already has
(another device, a restore, a duplicate) used to transfer every byte just
for the server to discard them as a dedup hit. The frontend now computes
the file's BLAKE3 locally and, on a hit, registers the file with a single
~150-byte metadata call.

Server — POST /api/files/by-hash:
- All checks live in the application service per the AuthZ rule:
  Create permission on the target folder via the authorization engine,
  hash ownership via the existing user-scoped query (a non-owned hash
  returns 404 — same shape as "no such blob" — and emits an
  instant_upload.rejected audit event), quota on the logical size.
- On success: one ref_count bump + the existing save_file_with_blob row
  registration (compensation included); is_new_blob=false so lifecycle
  hooks skip thumbnail regeneration. ~10 ms warm.
- The storage-usage service is now built before the application services
  and injected, instead of only living on AppState.

Client — WASM BLAKE3 + worker:
- wasm/oxicloud-hash: the exact same blake3 crate the server uses,
  compiled with WASM SIMD128 (~660 MB/s measured) so browser hashes match
  server content addresses bit for bit. Built by scripts/build-wasm.sh;
  the artifacts (45 KB wasm + 8 KB glue) are vendored like pdf.js — no
  npm dependencies, no wasm toolchain needed for regular builds.
- static/js/workers/hashWorker.js streams the File in 8 MiB slices off
  the main thread (constant RAM at any file size).
- features/files/instantUpload.js orchestrates: threshold (8 MiB — below
  it the round-trips cost more than the bytes), user-scoped
  /api/dedup/check, by-hash registration, and silent fallback to the
  normal byte upload on any miss, race or unsupported environment.
  Wired into both uploadFiles and uploadFolderEntries.
- biome.json vendors exclusion fixed to cover nested directories
  (previous vendors were .mjs and never matched the *.js include).

Verified end-to-end against PostgreSQL 16: node-driven WASM hash equals
the server's content_hash for a 20 MB file; by-hash returns 201 in ~10 ms
warm with a 151-byte request (vs 20,971,873 bytes for the byte upload);
the copy downloads byte-identical and the manifest ref_count goes 1→2;
a second user probing the same hash gets exists:false and 404 plus the
audit line; duplicate name → 409, malformed hash → 400; worker and wasm
are served with correct MIME (application/wasm).

https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS
This commit is contained in:
Claude
2026-06-11 13:54:32 +00:00
parent 944c833787
commit 0fab4ce17d
17 changed files with 1209 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
static/js/features/files/fileOperations.js
+67 -52
View File
@@ -12,6 +12,7 @@ import { i18n } from '../../core/i18n.js';
import { notifications } from '../../core/notifications.js';
import { invalidateFolderMeta } from '../../model/filesModel.js';
import { triggerBrowserDownload } from '../../utils/download.js';
import { tryInstantUpload } from './instantUpload.js';
/**
* @typedef {Object} BatchResult
@@ -404,20 +405,28 @@ const fileOps = {
if (quotaStop) return;
const file = readableFiles[idx];
const formData = new FormData();
if (targetFolderId) formData.append('folder_id', targetFolderId);
formData.append('file', file);
// ── Instant upload: when the server already has this exact
// content for this user, register it by hash — zero bytes
// on the wire. Any miss/failure falls back to a byte upload.
/** @type {UploadAnswer | null} */
let result = await tryInstantUpload(file, targetFolderId);
if (result) {
if (batchId) {
try {
notifications.updateFile(batchId, file.name, 100, result.ok ? 'done' : 'error');
} catch (_) {}
}
} else {
const formData = new FormData();
if (targetFolderId) formData.append('folder_id', targetFolderId);
formData.append('file', file);
console.log(`Uploading file to folder: ${targetFolderId || 'root'}`, {
file: file.name,
size: file.size
});
// Scale stall timeout with file size:
// base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit
const sizeGB = file.size / (1024 * 1024 * 1024);
const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000);
const result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout);
// Scale stall timeout with file size:
// base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit
const sizeGB = file.size / (1024 * 1024 * 1024);
const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000);
result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout);
}
uploadedCount++;
@@ -663,48 +672,54 @@ const fileOps = {
const parentPath = parts.slice(0, -1).join('/');
const targetFolderId = folderMap.get(parentPath) || currentFolderId;
// ── FIFO/pipe guard (0-byte files only) ──
// Named pipes (runit supervise/control) report size=0
// but block on open(). Pre-read only 0-byte files into
// memory; files with size>0 are always regular files and
// go straight to FormData (zero extra memory copy).
/** @type {Blob} */
let uploadFile = file; // default: use original File
if (file.size === 0) {
try {
const buf = await Promise.race([
file.arrayBuffer(),
new Promise((_, rej) => setTimeout(() => rej(new Error('read-timeout')), 2000))
]);
uploadFile = new Blob([buf], {
type: file.type || 'application/octet-stream'
});
} catch {
console.warn(`[SKIP] #${idx} ${rel} — cannot read 0-byte file (FIFO/pipe?), skipping`);
uploadedCount++;
successCount++;
if (batchId) {
try {
notifications.fileCompleted(batchId, true);
} catch (_) {}
// ── Instant upload (zero bytes on the wire) ──
// Same fallback contract as uploadFiles: a null result
// means "do the byte upload". The shared accounting
// after this try block handles both outcomes.
const instant = await tryInstantUpload(file, targetFolderId);
if (instant) {
result = instant;
} else {
// ── FIFO/pipe guard (0-byte files only) ──
// Named pipes (runit supervise/control) report size=0
// but block on open(). Pre-read only 0-byte files into
// memory; files with size>0 are always regular files and
// go straight to FormData (zero extra memory copy).
/** @type {Blob} */
let uploadFile = file; // default: use original File
if (file.size === 0) {
try {
const buf = await Promise.race([
file.arrayBuffer(),
new Promise((_, rej) => setTimeout(() => rej(new Error('read-timeout')), 2000))
]);
uploadFile = new Blob([buf], {
type: file.type || 'application/octet-stream'
});
} catch {
console.warn(`[SKIP] #${idx} ${rel} — cannot read 0-byte file (FIFO/pipe?), skipping`);
uploadedCount++;
successCount++;
if (batchId) {
try {
notifications.fileCompleted(batchId, true);
} catch (_) {}
}
return;
}
return;
}
const formData = new FormData();
formData.append('folder_id', targetFolderId);
formData.append('file', uploadFile, file.name);
const thisTimeout =
file.size === 0
? TIMEOUT_MS_ZERO
: Math.max(TIMEOUT_MIN_MS, TIMEOUT_BASE_MS + Math.ceil(file.size / (1024 * 1024)) * TIMEOUT_PER_MB_MS);
result = await this._uploadFileFetch(formData, thisTimeout);
}
const formData = new FormData();
formData.append('folder_id', targetFolderId);
formData.append('file', uploadFile, file.name);
const thisTimeout =
file.size === 0
? TIMEOUT_MS_ZERO
: Math.max(TIMEOUT_MIN_MS, TIMEOUT_BASE_MS + Math.ceil(file.size / (1024 * 1024)) * TIMEOUT_PER_MB_MS);
console.log(`[UPLOAD START] #${idx} ${rel} (${file.size} bytes, timeout=${thisTimeout}ms)`);
result = await this._uploadFileFetch(formData, thisTimeout);
console.log(`[UPLOAD END] #${idx} ${rel} ok=${result.ok}${result.errorMsg ? ` err=${result.errorMsg}` : ''}`);
} catch (e) {
result = {
ok: false,
static/js/features/files/instantUpload.js
+177
View File
@@ -0,0 +1,177 @@
/**
* OxiCloud - Instant upload (zero-byte dedup upload)
*
* Before transferring a file's bytes, compute its BLAKE3 locally (in a
* worker, off the main thread) and ask the server whether the caller
* already owns that exact content (`GET /api/dedup/check/{hash}` — the
* check is user-scoped, never a global content oracle). On a hit, a
* single metadata call (`POST /api/files/by-hash`) registers the file
* with ZERO content bytes on the wire.
*
* Performance posture:
* - Hashing runs in a dedicated worker with WASM SIMD128 — the UI thread
* never blocks, RAM stays constant (8 MiB slices).
* - Files below {@link INSTANT_UPLOAD_MIN_SIZE} skip the whole dance:
* two extra round-trips cost more than just uploading them.
* - Any failure (no WASM support, worker error, server miss, races)
* falls back silently to the normal byte upload — instant upload is
* an optimization, never a gate.
*/
import { getCsrfHeaders } from '../../core/csrf.js';
/**
* Files smaller than this upload normally: hashing + two round-trips
* outweigh the transfer. 8 MiB matches the chunked-upload threshold's
* order of magnitude.
*/
export const INSTANT_UPLOAD_MIN_SIZE = 8 * 1024 * 1024;
// Absolute URL on purpose — works in dev and in the release IIFE bundle
// (same pattern as the pdf.js loader in thumbnail.js).
const HASH_WORKER_URL = '/js/workers/hashWorker.js';
/** Hashing budget: 60 s base + 30 s per GB (WASM SIMD does ~0.5-1 GB/s). */
const HASH_TIMEOUT_BASE_MS = 60000;
const HASH_TIMEOUT_PER_GB_MS = 30000;
/**
* `false` once the environment proved unable to run the worker/WASM
* (old browser, blocked worker) — later files skip straight to the byte
* upload instead of failing the same way again. `null` = not yet known.
* @type {boolean | null}
*/
let _instantUploadUsable = null;
/**
* Hash a file in a one-shot worker. Resolves `null` on any failure —
* the caller falls back to a normal upload.
* @param {File} file
* @returns {Promise<string | null>}
*/
function hashFileInWorker(file) {
return new Promise((resolve) => {
/** @type {Worker} */
let worker;
try {
worker = new Worker(HASH_WORKER_URL, { type: 'module' });
} catch (_) {
_instantUploadUsable = false;
resolve(null);
return;
}
const sizeGB = file.size / (1024 * 1024 * 1024);
const timeoutMs = HASH_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * HASH_TIMEOUT_PER_GB_MS;
/** @param {string | null} hash */
const settle = (hash) => {
clearTimeout(timer);
worker.terminate();
resolve(hash);
};
const timer = setTimeout(() => settle(null), timeoutMs);
worker.onmessage = (event) => {
const data = /** @type {{ ok: boolean, hash?: string, error?: string }} */ (event.data);
if (!data.ok) {
// The worker ran but WASM failed (e.g. no SIMD128 support):
// a permanent environment property, don't retry per file.
_instantUploadUsable = false;
}
settle(data.ok && data.hash ? data.hash : null);
};
worker.onerror = () => {
// Worker script failed to load/parse — permanent.
_instantUploadUsable = false;
settle(null);
};
worker.postMessage({ file });
});
}
/**
* Ask the server whether the caller already owns content with this hash.
* @param {string} hash
* @returns {Promise<boolean>}
*/
async function callerOwnsHash(hash) {
try {
const response = await fetch(`/api/dedup/check/${hash}`, {
headers: { 'Cache-Control': 'no-cache, no-store, must-revalidate' }
});
if (!response.ok) return false;
const body = /** @type {import('../../core/types.js').HashCheckAnswer} */ (await response.json());
return body.exists === true;
} catch (_) {
return false;
}
}
/**
* Try to register `file` as a zero-byte instant upload.
*
* Returns `null` whenever the byte upload should proceed (file too
* small, environment unusable, hash miss, lost race, transient errors).
* Returns an upload-result object compatible with the uploaders'
* `UploadAnswer` shape when the attempt is conclusive — success, quota
* exceeded, or name conflict (a byte upload would fail identically).
*
* @param {File} file
* @param {string | null | undefined} folderId
* @returns {Promise<{ ok: boolean, data?: any, errorMsg?: string, isQuotaError?: boolean } | null>}
*/
export async function tryInstantUpload(file, folderId) {
if (!folderId || file.size < INSTANT_UPLOAD_MIN_SIZE || _instantUploadUsable === false || typeof Worker === 'undefined') {
return null;
}
const hash = await hashFileInWorker(file);
if (!hash) return null;
if (!(await callerOwnsHash(hash))) return null;
try {
const response = await fetch('/api/files/by-hash', {
method: 'POST',
headers: {
...getCsrfHeaders(),
'Content-Type': 'application/json',
'Cache-Control': 'no-cache, no-store, must-revalidate'
},
body: JSON.stringify(
/** @type {import('../../core/types.js').CreateFileByHash} */ ({
name: file.name,
folder_id: folderId,
hash
})
)
});
if (response.status === 201) {
return { ok: true, data: await response.json() };
}
/** @type {string} */
let errorMsg = `Instant upload failed (HTTP ${response.status})`;
try {
const body = await response.json();
errorMsg = body.message || body.error || errorMsg;
} catch (_) {}
if (response.status === 507) {
return { ok: false, isQuotaError: true, errorMsg };
}
if (response.status === 409) {
// Duplicate name in the folder — a byte upload would hit the
// exact same conflict; surface it without transferring.
return { ok: false, errorMsg };
}
// 404 (ownership race with a delete+GC), 4xx/5xx: fall back to the
// byte upload — the server dedups it on write anyway.
return null;
} catch (_) {
return null;
}
}