From 5d034b0d09a399b85597c47ee0a4e19b1162bc79 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 15:44:44 +0000 Subject: [PATCH] Delta-upload client: FastCDC in WASM + overlapped worker pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 — the client side of "upload only what changed", closing the delta-sync plan. WASM (wasm/oxicloud-hash): DeltaChunker adds incremental FastCDC with the server's exact crate and parameters (64K/256K/1M) next to the BLAKE3 hasher. The incremental split is provably identical to a single pass: every chunk except the last ends on a content/max-size condition whose decision window was fully buffered, so only the tail is provisional and re-examined as slices arrive. A mirror test — the client twin of the server's stream≡slice test — chunks 4 MiB of xorshift noise with adversarial slice sizes (7 B … 8 MiB) and requires boundary-for-boundary equality with one FastCDC pass. Vendored artifacts rebuilt (55 KB wasm). Worker (static/js/workers/deltaWorker.js): the full protocol off the main thread with OVERLAPPED stages — 8 MiB file slices feed the chunker while earlier batches (256 hashes) negotiate and their missing chunks upload through a 2-deep PUT pool (≤8 MiB framed bodies, bytes re-sliced from the File at send time, never hoarded). Commit handles 409 still_missing by uploading exactly the named hashes and retrying. Orchestrator (features/files/deltaUpload.js): threshold (8 MiB), worker lifecycle + size-scaled timeout, progress relay to the upload bell, conclusive-outcome mapping (201/200, 507 quota, 409 name conflict) and silent fallback to the byte upload for everything else. Wired into uploadFiles and uploadFolderEntries, which now surface one batch summary of the bytes dedup saved. This subsumes the whole-file instant-upload module — a fully-known file negotiates to nothing missing and the commit short-circuits on possession — so instantUpload.js and hashWorker.js are removed (the /api/dedup/check and /api/files/by-hash endpoints remain for API clients). Verified end-to-end against PostgreSQL 16 — the cross-boundary proof the whole design hangs on, in both directions: a 24 MB file byte- uploaded (server-side CDC) then edited and delta-negotiated with WASM-computed chunks reported missing 1/74 (boundaries bit-identical), synced with 344 KB on the wire vs 24 MB (98.6% saved) and downloaded byte-identical; inversely, a file created via delta then byte-uploaded as identical content produced a server-side manifest DEDUP HIT with the same content_hash. Insertion at the head of the file (the adversarial CDC case) still negotiated missing 1/74. Chunk+hash throughput ≈275 MB/s in V8 with SIMD128. https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS --- docs/delta-upload-protocol.md | 19 +- static/js/features/files/deltaUpload.js | 160 +++++++++ static/js/features/files/fileOperations.js | 46 ++- static/js/features/files/instantUpload.js | 177 ---------- .../vendors/hash-wasm/oxicloud_hash_wasm.js | 148 ++++++-- .../hash-wasm/oxicloud_hash_wasm_bg.wasm | Bin 45242 -> 55809 bytes static/js/workers/deltaWorker.js | 330 ++++++++++++++++++ static/js/workers/hashWorker.js | 79 ----- wasm/oxicloud-hash/Cargo.lock | 7 + wasm/oxicloud-hash/Cargo.toml | 4 + wasm/oxicloud-hash/src/lib.rs | 221 ++++++++++++ 11 files changed, 885 insertions(+), 306 deletions(-) create mode 100644 static/js/features/files/deltaUpload.js delete mode 100644 static/js/features/files/instantUpload.js create mode 100644 static/js/workers/deltaWorker.js delete mode 100644 static/js/workers/hashWorker.js diff --git a/docs/delta-upload-protocol.md b/docs/delta-upload-protocol.md index 31fd58ff..171e1a1d 100644 --- a/docs/delta-upload-protocol.md +++ b/docs/delta-upload-protocol.md @@ -12,15 +12,20 @@ Editing a few bytes of a 500 MB file re-uploads ~1 MiB instead of ## Who can use it -Any authenticated API client. The OxiCloud web frontend adopts it in a -later phase; generic WebDAV/NextCloud clients cannot (their protocols -have no delta concept) — they keep uploading full bytes, and the server -keeps deduplicating those on write. +Any authenticated API client. The OxiCloud web frontend uses it +automatically for files ≥ 8 MiB (`features/files/deltaUpload.js` + +`workers/deltaWorker.js`, chunking with the vendored WASM build of the +server's own FastCDC+BLAKE3 crates, falling back to a plain byte upload +on any failure). Generic WebDAV/NextCloud clients cannot (their +protocols have no delta concept) — they keep uploading full bytes, and +the server keeps deduplicating those on write. Chunk boundaries are the **client's choice**: matching the server's -FastCDC parameters maximizes cross-version sharing, but any split with -chunks of 1 byte … 1 MiB is valid — correctness is guaranteed by -server-side verification, not by the chunking scheme. +FastCDC parameters (64 KB / 256 KB / 1 MiB, as the bundled WASM module +does) maximizes cross-version sharing — including against versions that +entered through plain byte uploads — but any split with chunks of +1 byte … 1 MiB is valid; correctness is guaranteed by server-side +verification, not by the chunking scheme. ## The three steps diff --git a/static/js/features/files/deltaUpload.js b/static/js/features/files/deltaUpload.js new file mode 100644 index 00000000..f6a17d14 --- /dev/null +++ b/static/js/features/files/deltaUpload.js @@ -0,0 +1,160 @@ +/** + * OxiCloud - Delta upload ("upload only what changed"). + * + * Main-thread orchestrator for `workers/deltaWorker.js`, which runs the + * whole client side of the delta protocol off the UI thread: FastCDC + * chunking + BLAKE3 (the same WASM crate and parameters as the server, + * so boundaries match bit for bit), per-batch negotiation, upload of + * only the missing chunks, and the commit. + * + * This SUBSUMES the previous whole-file instant upload: a fully known + * file negotiates to "nothing missing" and the commit short-circuits on + * possession of the file hash — same zero-byte outcome, one pipeline. + * + * Performance posture: + * - Stages overlap inside the worker (hash ‖ negotiate ‖ upload), so + * wall-clock approaches max(hash, upload) instead of their sum. + * - RAM stays flat: 8 MiB read slices; chunk bytes are re-sliced from + * the File at upload time, never hoarded. + * - Files below {@link DELTA_UPLOAD_MIN_SIZE} skip the pipeline: the + * round-trips cost more than the bytes. + * - Any failure falls back silently to the normal byte upload — delta + * is an optimization, never a gate. + */ + +import { getCsrfToken } from '../../core/csrf.js'; + +/** + * Files smaller than this upload normally: hashing + negotiation + * round-trips outweigh the transfer. + */ +export const DELTA_UPLOAD_MIN_SIZE = 8 * 1024 * 1024; + +// Absolute URL on purpose — works in dev and in the release IIFE bundle +// (same pattern as the pdf.js loader in thumbnail.js). +const DELTA_WORKER_URL = '/js/workers/deltaWorker.js'; + +/** Budget: 120 s base + 90 s per GB (hashing + uploading the delta). */ +const DELTA_TIMEOUT_BASE_MS = 120000; +const DELTA_TIMEOUT_PER_GB_MS = 90000; + +/** + * `false` once the environment proved unable to run the worker/WASM — + * later files skip straight to the byte upload. `null` = not yet known. + * @type {boolean | null} + */ +let _deltaUploadUsable = null; + +/** + * Result contract shared with the uploaders' `UploadAnswer`, plus the + * bandwidth accounting the UI surfaces. + * @typedef {Object} DeltaUploadAnswer + * @property {boolean} ok + * @property {any} [data] FileDto on success + * @property {string} [errorMsg] + * @property {boolean} [isQuotaError] + * @property {number} [savedBytes] bytes NOT transferred thanks to dedup + */ + +/** + * Try to upload `file` through the delta protocol. + * + * Resolves `null` whenever the plain byte upload should proceed (file too + * small, environment unusable, any transport/protocol failure). Resolves + * a {@link DeltaUploadAnswer} when the outcome is conclusive — success, + * quota exceeded, or a name conflict a byte upload would also hit. + * + * @param {File} file + * @param {string | null | undefined} folderId + * @param {(pct: number) => void} [onProgress] 0-99 while transferring + * @returns {Promise} + */ +export function tryDeltaUpload(file, folderId, onProgress) { + if (!folderId || file.size < DELTA_UPLOAD_MIN_SIZE || _deltaUploadUsable === false || typeof Worker === 'undefined') { + return Promise.resolve(null); + } + + return new Promise((resolve) => { + /** @type {Worker} */ + let worker; + try { + worker = new Worker(DELTA_WORKER_URL, { type: 'module' }); + } catch (_) { + _deltaUploadUsable = false; + resolve(null); + return; + } + + const sizeGB = file.size / (1024 * 1024 * 1024); + const timeoutMs = DELTA_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * DELTA_TIMEOUT_PER_GB_MS; + + let savedBytes = 0; + + /** @param {DeltaUploadAnswer | null} answer */ + const settle = (answer) => { + clearTimeout(timer); + worker.terminate(); + resolve(answer); + }; + const timer = setTimeout(() => settle(null), timeoutMs); + + worker.onmessage = (event) => { + const msg = /** @type {any} */ (event.data); + if (msg.type === 'progress') { + savedBytes = msg.reusedBytes; + if (onProgress && msg.totalBytes > 0) { + const pct = Math.min(99, Math.round((100 * (msg.reusedBytes + msg.uploadedBytes)) / msg.totalBytes)); + onProgress(pct); + } + return; + } + if (msg.type === 'fallback') { + settle(null); + return; + } + if (msg.type === 'done') { + if (msg.status === 201 || msg.status === 200) { + settle({ ok: true, data: msg.body, savedBytes }); + return; + } + /** @type {string} */ + const errorMsg = msg.body?.message || msg.body?.error || `Delta upload failed (HTTP ${msg.status})`; + if (msg.status === 507) { + settle({ ok: false, isQuotaError: true, errorMsg }); + return; + } + if (msg.status === 409 && !msg.body?.still_missing) { + // Duplicate name — a byte upload would hit the same wall. + settle({ ok: false, errorMsg }); + return; + } + // still_missing exhausted, 4xx/5xx oddities: byte upload is + // the safe road (the server dedups it on write anyway). + settle(null); + } + }; + worker.onerror = () => { + // Worker script failed to load/parse — permanent environment trait. + _deltaUploadUsable = false; + settle(null); + }; + + worker.postMessage({ + file, + folderId, + name: file.name, + csrfToken: getCsrfToken() || '' + }); + }); +} + +/** + * Bilingual one-line summary for the bandwidth saved by a batch. + * @param {number} savedBytes + * @param {string} locale + * @returns {string} + */ +export function formatSavedSummary(savedBytes, locale) { + const mb = (savedBytes / (1024 * 1024)).toFixed(1); + return locale.startsWith('es') ? `Deduplicación: ${mb} MB no necesitaron subirse` : `Deduplication: ${mb} MB didn't need uploading`; +} diff --git a/static/js/features/files/fileOperations.js b/static/js/features/files/fileOperations.js index a4bc092d..f62b52d3 100644 --- a/static/js/features/files/fileOperations.js +++ b/static/js/features/files/fileOperations.js @@ -12,7 +12,7 @@ import { i18n } from '../../core/i18n.js'; import { notifications } from '../../core/notifications.js'; import { invalidateFolderMeta } from '../../model/filesModel.js'; import { triggerBrowserDownload } from '../../utils/download.js'; -import { tryInstantUpload } from './instantUpload.js'; +import { formatSavedSummary, tryDeltaUpload } from './deltaUpload.js'; /** * @typedef {Object} BatchResult @@ -392,6 +392,7 @@ const fileOps = { let uploadedCount = 0; let successCount = 0; let quotaStop = false; + let savedBytesTotal = 0; const targetFolderId = app.currentPath || app.userHomeFolderId; @@ -405,12 +406,19 @@ const fileOps = { if (quotaStop) return; const file = readableFiles[idx]; - // ── Instant upload: when the server already has this exact - // content for this user, register it by hash — zero bytes - // on the wire. Any miss/failure falls back to a byte upload. - /** @type {UploadAnswer | null} */ - let result = await tryInstantUpload(file, targetFolderId); + // ── Delta upload: chunk + hash locally (worker/WASM) and + // transfer only what the server doesn't already have for + // this user. Any miss/failure falls back to a byte upload. + /** @type {UploadAnswer & { savedBytes?: number } | null} */ + let result = await tryDeltaUpload(file, targetFolderId, (pct) => { + if (batchId) { + try { + notifications.updateFile(batchId, file.name, pct, 'uploading'); + } catch (_) {} + } + }); if (result) { + savedBytesTotal += result.savedBytes || 0; if (batchId) { try { notifications.updateFile(batchId, file.name, 100, result.ok ? 'done' : 'error'); @@ -492,6 +500,14 @@ const fileOps = { // All done this._finishUploadToast(successCount, totalFiles); + if (savedBytesTotal > 0 && notifications) { + notifications.addNotification({ + icon: 'fa-bolt', + iconClass: 'upload', + title: i18n?.getCurrentLocale?.()?.startsWith('es') ? 'Subida delta' : 'Delta upload', + text: formatSavedSummary(savedBytesTotal, i18n?.getCurrentLocale?.() || 'en') + }); + } // Refresh storage usage display try { @@ -643,6 +659,7 @@ const fileOps = { let uploadedCount = 0; let successCount = 0; let quotaStop = false; + let savedBytesTotal = 0; // ── Concurrent upload with limited parallelism ────────── // FIFOs are pre-caught by the 0-byte arrayBuffer guard, @@ -672,13 +689,14 @@ const fileOps = { const parentPath = parts.slice(0, -1).join('/'); const targetFolderId = folderMap.get(parentPath) || currentFolderId; - // ── Instant upload (zero bytes on the wire) ── + // ── Delta upload (only changed bytes on the wire) ── // Same fallback contract as uploadFiles: a null result // means "do the byte upload". The shared accounting // after this try block handles both outcomes. - const instant = await tryInstantUpload(file, targetFolderId); - if (instant) { - result = instant; + const delta = await tryDeltaUpload(file, targetFolderId); + if (delta) { + result = delta; + savedBytesTotal += delta.savedBytes || 0; } else { // ── FIFO/pipe guard (0-byte files only) ── // Named pipes (runit supervise/control) report size=0 @@ -773,6 +791,14 @@ const fileOps = { await Promise.all(workers); this._finishUploadToast(successCount, totalFiles); + if (savedBytesTotal > 0 && notifications) { + notifications.addNotification({ + icon: 'fa-bolt', + iconClass: 'upload', + title: i18n?.getCurrentLocale?.()?.startsWith('es') ? 'Subida delta' : 'Delta upload', + text: formatSavedSummary(savedBytesTotal, i18n?.getCurrentLocale?.() || 'en') + }); + } try { await refreshUserData(); diff --git a/static/js/features/files/instantUpload.js b/static/js/features/files/instantUpload.js deleted file mode 100644 index bda924e1..00000000 --- a/static/js/features/files/instantUpload.js +++ /dev/null @@ -1,177 +0,0 @@ -/** - * OxiCloud - Instant upload (zero-byte dedup upload) - * - * Before transferring a file's bytes, compute its BLAKE3 locally (in a - * worker, off the main thread) and ask the server whether the caller - * already owns that exact content (`GET /api/dedup/check/{hash}` — the - * check is user-scoped, never a global content oracle). On a hit, a - * single metadata call (`POST /api/files/by-hash`) registers the file - * with ZERO content bytes on the wire. - * - * Performance posture: - * - Hashing runs in a dedicated worker with WASM SIMD128 — the UI thread - * never blocks, RAM stays constant (8 MiB slices). - * - Files below {@link INSTANT_UPLOAD_MIN_SIZE} skip the whole dance: - * two extra round-trips cost more than just uploading them. - * - Any failure (no WASM support, worker error, server miss, races) - * falls back silently to the normal byte upload — instant upload is - * an optimization, never a gate. - */ - -import { getCsrfHeaders } from '../../core/csrf.js'; - -/** - * Files smaller than this upload normally: hashing + two round-trips - * outweigh the transfer. 8 MiB matches the chunked-upload threshold's - * order of magnitude. - */ -export const INSTANT_UPLOAD_MIN_SIZE = 8 * 1024 * 1024; - -// Absolute URL on purpose — works in dev and in the release IIFE bundle -// (same pattern as the pdf.js loader in thumbnail.js). -const HASH_WORKER_URL = '/js/workers/hashWorker.js'; - -/** Hashing budget: 60 s base + 30 s per GB (WASM SIMD does ~0.5-1 GB/s). */ -const HASH_TIMEOUT_BASE_MS = 60000; -const HASH_TIMEOUT_PER_GB_MS = 30000; - -/** - * `false` once the environment proved unable to run the worker/WASM - * (old browser, blocked worker) — later files skip straight to the byte - * upload instead of failing the same way again. `null` = not yet known. - * @type {boolean | null} - */ -let _instantUploadUsable = null; - -/** - * Hash a file in a one-shot worker. Resolves `null` on any failure — - * the caller falls back to a normal upload. - * @param {File} file - * @returns {Promise} - */ -function hashFileInWorker(file) { - return new Promise((resolve) => { - /** @type {Worker} */ - let worker; - try { - worker = new Worker(HASH_WORKER_URL, { type: 'module' }); - } catch (_) { - _instantUploadUsable = false; - resolve(null); - return; - } - - const sizeGB = file.size / (1024 * 1024 * 1024); - const timeoutMs = HASH_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * HASH_TIMEOUT_PER_GB_MS; - - /** @param {string | null} hash */ - const settle = (hash) => { - clearTimeout(timer); - worker.terminate(); - resolve(hash); - }; - const timer = setTimeout(() => settle(null), timeoutMs); - - worker.onmessage = (event) => { - const data = /** @type {{ ok: boolean, hash?: string, error?: string }} */ (event.data); - if (!data.ok) { - // The worker ran but WASM failed (e.g. no SIMD128 support): - // a permanent environment property, don't retry per file. - _instantUploadUsable = false; - } - settle(data.ok && data.hash ? data.hash : null); - }; - worker.onerror = () => { - // Worker script failed to load/parse — permanent. - _instantUploadUsable = false; - settle(null); - }; - - worker.postMessage({ file }); - }); -} - -/** - * Ask the server whether the caller already owns content with this hash. - * @param {string} hash - * @returns {Promise} - */ -async function callerOwnsHash(hash) { - try { - const response = await fetch(`/api/dedup/check/${hash}`, { - headers: { 'Cache-Control': 'no-cache, no-store, must-revalidate' } - }); - if (!response.ok) return false; - const body = /** @type {import('../../core/types.js').HashCheckAnswer} */ (await response.json()); - return body.exists === true; - } catch (_) { - return false; - } -} - -/** - * Try to register `file` as a zero-byte instant upload. - * - * Returns `null` whenever the byte upload should proceed (file too - * small, environment unusable, hash miss, lost race, transient errors). - * Returns an upload-result object compatible with the uploaders' - * `UploadAnswer` shape when the attempt is conclusive — success, quota - * exceeded, or name conflict (a byte upload would fail identically). - * - * @param {File} file - * @param {string | null | undefined} folderId - * @returns {Promise<{ ok: boolean, data?: any, errorMsg?: string, isQuotaError?: boolean } | null>} - */ -export async function tryInstantUpload(file, folderId) { - if (!folderId || file.size < INSTANT_UPLOAD_MIN_SIZE || _instantUploadUsable === false || typeof Worker === 'undefined') { - return null; - } - - const hash = await hashFileInWorker(file); - if (!hash) return null; - - if (!(await callerOwnsHash(hash))) return null; - - try { - const response = await fetch('/api/files/by-hash', { - method: 'POST', - headers: { - ...getCsrfHeaders(), - 'Content-Type': 'application/json', - 'Cache-Control': 'no-cache, no-store, must-revalidate' - }, - body: JSON.stringify( - /** @type {import('../../core/types.js').CreateFileByHash} */ ({ - name: file.name, - folder_id: folderId, - hash - }) - ) - }); - - if (response.status === 201) { - return { ok: true, data: await response.json() }; - } - - /** @type {string} */ - let errorMsg = `Instant upload failed (HTTP ${response.status})`; - try { - const body = await response.json(); - errorMsg = body.message || body.error || errorMsg; - } catch (_) {} - - if (response.status === 507) { - return { ok: false, isQuotaError: true, errorMsg }; - } - if (response.status === 409) { - // Duplicate name in the folder — a byte upload would hit the - // exact same conflict; surface it without transferring. - return { ok: false, errorMsg }; - } - // 404 (ownership race with a delete+GC), 4xx/5xx: fall back to the - // byte upload — the server dedups it on write anyway. - return null; - } catch (_) { - return null; - } -} diff --git a/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js b/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js index 2a341978..b706d36b 100644 --- a/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js +++ b/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js @@ -70,6 +70,99 @@ export class Blake3Hasher { } if (Symbol.dispose) Blake3Hasher.prototype[Symbol.dispose] = Blake3Hasher.prototype.free; +/** + * Incremental FastCDC chunker + whole-file BLAKE3, for the delta-upload + * worker. Feed the file in slices; every call returns the chunks that + * became FINAL; `finish()` flushes the tail and returns the file hash. + * + * ```js + * const c = new DeltaChunker(); + * for (const slice of slices) { + * for (const [h, s] of JSON.parse(c.update(bytes))) { … } + * } + * const { chunks, file_hash } = JSON.parse(c.finish()); + * ``` + * + * Correctness of the incremental split: FastCDC decides each cut by + * scanning at most `CDC_MAX_CHUNK` bytes from the chunk's start. When + * the chunker runs over the buffered prefix of a longer file, every + * produced chunk except the LAST ended on a content/max-size condition + * — its decision window was fully available, so the full-file chunker + * makes the same cut. Only the last chunk (cut by "end of buffer") is + * provisional: it stays buffered and is re-examined when more bytes + * arrive. By induction the emitted boundaries equal a single FastCDC + * pass over the whole file — the mirror test below proves it. + */ +export class DeltaChunker { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + DeltaChunkerFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_deltachunker_free(ptr, 0); + } + /** + * Flush the provisional tail and return + * `{"chunks":[["",size]…],"file_hash":"","total":N}`. + * `chunks` holds at most one entry (the tail); an empty file has none + * and its `file_hash` is BLAKE3 of the empty input. + * @returns {string} + */ + finish() { + let deferred1_0; + let deferred1_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.deltachunker_finish(retptr, this.__wbg_ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred1_0 = r0; + deferred1_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred1_0, deferred1_1, 1); + } + } + /** + * Create a chunker with the server's CDC parameters. + */ + constructor() { + const ret = wasm.deltachunker_new(); + this.__wbg_ptr = ret; + DeltaChunkerFinalization.register(this, this.__wbg_ptr, this); + return this; + } + /** + * Feed one slice. Returns a JSON array of the chunks that became + * final: `[["", size], …]` (possibly empty). + * @param {Uint8Array} data + * @returns {string} + */ + update(data) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.deltachunker_update(retptr, this.__wbg_ptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred2_0, deferred2_1, 1); + } + } +} +if (Symbol.dispose) DeltaChunker.prototype[Symbol.dispose] = DeltaChunker.prototype.free; + /** * One-shot convenience for small buffers. * @param {Uint8Array} data @@ -96,28 +189,26 @@ export function blake3Hex(data) { function __wbg_get_imports() { const import0 = { __proto__: null, - __wbg___wbindgen_throw_bbadd78c1bac3a77: (arg0, arg1) => { + __wbg___wbindgen_throw_bbadd78c1bac3a77: function(arg0, arg1) { throw new Error(getStringFromWasm0(arg0, arg1)); - } + }, }; return { __proto__: null, - './oxicloud_hash_wasm_bg.js': import0 + "./oxicloud_hash_wasm_bg.js": import0, }; } -const Blake3HasherFinalization = - typeof FinalizationRegistry === 'undefined' - ? { register: () => {}, unregister: () => {} } - : new FinalizationRegistry((ptr) => wasm.__wbg_blake3hasher_free(ptr, 1)); +const Blake3HasherFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_blake3hasher_free(ptr, 1)); +const DeltaChunkerFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_deltachunker_free(ptr, 1)); let cachedDataViewMemory0 = null; function getDataViewMemory0() { - if ( - cachedDataViewMemory0 === null || - cachedDataViewMemory0.buffer.detached === true || - (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer) - ) { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { cachedDataViewMemory0 = new DataView(wasm.memory.buffer); } return cachedDataViewMemory0; @@ -177,13 +268,9 @@ async function __wbg_load(module, imports) { const validResponse = module.ok && expectedResponseType(module.type); if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { - console.warn( - '`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n', - e - ); - } else { - throw e; - } + console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); + + } else { throw e; } } } @@ -201,10 +288,7 @@ async function __wbg_load(module, imports) { function expectedResponseType(type) { switch (type) { - case 'basic': - case 'cors': - case 'default': - return true; + case 'basic': case 'cors': case 'default': return true; } return false; } @@ -213,11 +297,12 @@ async function __wbg_load(module, imports) { function initSync(module) { if (wasm !== undefined) return wasm; + if (module !== undefined) { if (Object.getPrototypeOf(module) === Object.prototype) { - ({ module } = module); + ({module} = module) } else { - console.warn('using deprecated parameters for `initSync()`; pass a single object instead'); + console.warn('using deprecated parameters for `initSync()`; pass a single object instead') } } @@ -232,11 +317,12 @@ function initSync(module) { async function __wbg_init(module_or_path) { if (wasm !== undefined) return wasm; + if (module_or_path !== undefined) { if (Object.getPrototypeOf(module_or_path) === Object.prototype) { - ({ module_or_path } = module_or_path); + ({module_or_path} = module_or_path) } else { - console.warn('using deprecated parameters for the initialization function; pass a single object instead'); + console.warn('using deprecated parameters for the initialization function; pass a single object instead') } } @@ -245,11 +331,7 @@ async function __wbg_init(module_or_path) { } const imports = __wbg_get_imports(); - if ( - typeof module_or_path === 'string' || - (typeof Request === 'function' && module_or_path instanceof Request) || - (typeof URL === 'function' && module_or_path instanceof URL) - ) { + if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { module_or_path = fetch(module_or_path); } @@ -258,4 +340,4 @@ async function __wbg_init(module_or_path) { return __wbg_finalize_init(instance, module); } -export { __wbg_init as default, initSync }; +export { initSync, __wbg_init as default }; diff --git a/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm b/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm index 7ab00d463942f1b339a82028fd04fccc5e01d2c5..0f09d2a67647f0a5d3bed763fb86391889bf01c4 100644 GIT binary patch delta 21014 zcmcJ12Ut{B)BnA9E9@?;Vn^Zb+E78Tfg&mwMa3R_>Yo(NRq^JtH##|fctR2vSp2`*YIC(*O7on=8;Ugzuiv1V)Zw$vHS1ZLhXn;&UGKaSy61!l zrD_Wcb8VgLh6%5B+A`|y5YlsPZR!OXQ+MZaF;-sWr1agkdG)G#XXJ*qHR(mpBuJbO z=7d_gaHJv|XZZL{!1A-4wO>8*Dtl**jfJaWlCT?Cv z!SG?}1tDc?C{(}VKaj0a$VO8U-5Q1NW+73Y;6WwaXUP$Y0(` zi=j^7+M0N9SkwdxXeD01_YM?t8B~3s@Maf*N9QXlFqVP&yQ!QGF$)w{urjd{JI%@& zteojICWDn%v-Zs>%bcVPTFp8(u#TNhgEp{oHfulNG$xysb6NXQr!l##oX6TvJB`U> zWneay6;7}Lmv_sAzB2lRQ^$E$KFr^gav+@SkzLS-eF&kJpo3$Ts8k5b+xvc%D)0kXV&SUMToyO#`@(F5} zh09KZPO$QMHsR)nLBe@f&S&j+oyO#|asg|9;xwj!l^?S97fxdyvT`A7f5XZ>l|hB9 z@``m#C4rZeL9bYQN-kqNYgfjkFr|PlK%FpH8IZ~v^-_1HjnLZVl9!O3Yr7w6DbLhQ z)R2}%KqXN{MIU3)k{(u+{Gn8~&Qg$>!4v-*#(v4SI`r4hmCu z+lqyC>*!_FI~k%-t5$+VU^ZE!*krA!vD;)wS&?<(ELyTotOv>t1+3ikDACf570 zX`Rw57>BVK7Pi7H##nJ8WaipVg_ko}IIL+$q;0dZiq)3D0@&#h&8nDTNx>}ui?L#_ zR_EH*Hw%&(9LYU~EI$q-J3JbQp|Og=4aMGA`52rzG!ZC>V3x!KeKn1Wk$h01jZaCz z8IheEYE+Ts#L8>~9YCoMFeL0S3Mh{auS~}N%W#kXfog9# z+sqr~idbBTwW=#|WZC~W|3otXbm)RMIz-`FFdpEj#qlZfXfX-smjeYFrjHQ?QDxSZ z10UgFrMknh z>?NwhQYK?Z=s!6~=uAEHsi%Xeerx17p5r(*e$4pcQr;TWp%{Vg`mDQPomxM77Mb;kQNmu$hCd7AA>F|APkkw|OVy8Gz2vP8=tF$e24-^6bHkFUTZ*z>& zP>c{DSu~-sYD{aBi$OG)9njDJ4fM$rVA3`v~h?C|Zp@4*BApj`y61E!6+928nQjr24Tk?d4 z(5^BGOy$jb6PEJ9R3;y4h=-4Gm$yR2d%#1^fm=q9O?tgZtr>6k^E^!U&T*SX}@F##k#mj3z}!?nd=rGRc_` zh$}cRrOi}?44|>iZQu`(1bw259jvZ)vRsHPJtGT>4GKL01FEQ%vF8Z(L>EGh0MnkU zA=g9~Ab0<3$i4mRShT4uSRc)$S$mK`?KMMCW(7_npY z)q`>R80iN(NdTN3L4ut|J<+IRG{Ve)Tu+=q5he}BL3k-@RWa&d@C2d`>%>@O#KN>C zbZ*iSX+K0|I1H#_Dvy~Ob7iu6K$4C`+I@IGW}_8M127!5iYMVh0O%l#sl*|={U>2& z{h=SL!$&*)PXGvsCOM7T8O++eStJg&c75J9cej{GjMC5|c}UsR(GdCpyn&f?o(+U? zLYhEI5s7B$#T*Q+0j<6{7llF^Q9EaT-_O;&N6zEOX1rWbG)c-mqX1RO!6cJ?y>mpZ652IgINFu5o_cU<>>$KW>Oc!+g?es(zMYqADy! zFPg_rx0-`gdf2pL6T)1li{|lcq*b#e>^)z8N|<&!17KBCeaiR=qOhe0fT~1yTW0eb zvOBg`AG2jD92uS(Q8lw_3NL*5j+#;PV6fqII9 z1dP=1#^5TcBRs7dvj-3tL`|$!AfHNsb)KEW_-)m`l^qAZ(uI?hyoI8}RB08dRs6Iy zfLa5bTH&T}s!+HOc|-|S$d^cg(f5Pvu2UEY{QS6BoXU)=t}SUK*lBeSQWelN*I@1; zeO-dNpXrN)O?ZMNBv^I9+#@sqJN!@7D$v!$NVcrauchZ8SMX0~W_v^f6?!53)G8Q6 z)mu(2T8gu;+2VnN$=zI1U}h+Thv^JLssoMWHo&CO!Q~=pl478^#S!pq#(^Yyps>QW ztz|hGP4JUc(rTP%j&inB`8>+dC?}sv#m0dj3d->cr-*tBd|0VKajX=iQFEAAO^k(y z#B&Y#!Gv)b52v@EQO2Ro01D{@knn=U*!4l#wkM<$mq~DN%79Ekngk0ENs(5-d|*GM zLhOqx34T9|8S8?z6l4J~lQ}B8*vlf0N|HSx6uyNSa^BhEF3AANj~XBvY75aw|4>uVfQYuIG#Q8zq@9GOx6QMamz}ZNBBqk~L)H`? z!-5qFCmrcla)z7*wur~XFeXXR&H`c)=g-G9GH!CE$`3wg#?goShtv9b7#(Qcv{4v<|N z0KvJ{t_>-IT^$N}=mSIpqF^p5Qb5RCHKY*W8%3OoLg-K=h0qX}1v8EunFNm`qgcdM z=$o0eJBEJ6n#KPOSZFAh!oM|ew5UUYQ9v_n30f5tNefcP6Qc45OerG%3rt}t>~k4J z3I-f4bN~{?qWgH-hJFHc1a=c#gm947p$5wdquScU8#Uxsph=L9GGWC@q!N9I5h(~v znFba*X&`NxlLp$#wyh=?sV5}JsgEE5Eh6{>dSqP4%n|Y`^9RlTuc@%?#1~*|_JHLl zdtxq05Yy>27dgWtlVFl{AEg38t$$+2` zM+&Tzq>DjvfSZvL8puIUC9WEmk`fw(vLG2K1_p!3fnYTblgVFXk%!_i;?RZ|C}D&a zF@zu)5pGi>LI{Ck-ege-OOfIKA%ciUlM+CrfEY&rF%@NWz*Gea<5746hyt-nfNzBe zLPLdk3)>kVAqldHdNTYA3LMzUsBw0X6)(%-AgkeJ=>&Mv%)h0e(;m-h$jyRrVy-X% za?o6S^cFanl#3v12=Q0PN^Y&ee(*s;!a$-tgJT%JYM3WA)P)!~TVb4r@w6JZz|KdS z2$v0P1UDJGv}RY7{*qDu7e46-u_Ty%eBv4UC?qp>$>jX8oC)Y49dgj2MyP~QL^Y91 zf<^@4P}V4?6C1B!j{yo3?cCOU2uC#J(`w+igWE0+Kopk~v%zPXLJ_oMDMCbYaj=x4 ze>+Q^P+>SIOkL(+Dm-h>pIeg>+K^Lr!=Nc8c7PlJ z7g!u6c0ghhET`-d#SYGKw1I`r_Q+R|=D{NVW#lV6ht(g&5&t~*Z>Q4v{x^~PM@xVv znFdZXs0=c%gaxY+76?jMAcTS&$Tz0*%$ZTD?9fqxDNJLr4LoUXksHCx6VDDBdpb=Z zhk?g0r!fK4!u)5MOgxD#kM2;EvAwLUwDaUXYeam8AYXw`EVLM(!4?&bgT2Sug_0_m z8;lPeWf(^i!SDnmmW701ZU_*C0VS(xcgx{Iz^W?CspWWJ;+kOLvF$t}Fd|YR6pO$@ z(8^YI0!1Ap7ScpYp&+e_^c^Ksr0g^}Z*_5s+3o%RmKo$ne&EJGFazYmn*vJY!ZK#S z6ibTp%%}t~hw}`@2#`pTvt)p-yvsS}_&D&9~0`YUDkFls#5Z%G0)wFSLC?NcC5b2;CJ&t{6d_mn{Um#vYav7%> zs1RX}ZbFK9Q6@qxdg45mBIeskhE6%k0yIKN#@NXPm&0&cDgN~*ZSbx?CD;Z0`x`Pf zviN_b4t9gFkMy`7QKVy=Y+@;IYy^?)K(WvsSPFIn0!2b5T|{*VCJHPzVtVYLoMxgN z3Lzl)dS)j``H@4kD;!cGkPol|O?aPy41Xh71Ik+EeJ~9q4`3)82?hZp$N~hOTG7c$?%WFT^F78h2~>QXUEb53$g7ynUT)H5Y&sGOKcr(=^ zG`Nw$_3PG=bw0kAQ&t7RFGqSe$0H7?C@oK~S_I`r6U6@I4XTP|t0a2g;kZFXJV!U0 zJxARxOD`&L@B?9#M2?cggkH*STbI!_O=xq6n%i5_N7pP)qu>R=831+2{^Lsd{OJCC zX#@EDc27&sLvT0yYHV9I{7b>c%Z&*qNiI$6_K>SKDEHx% zBVP$TC0;^klno+rVr9EHk#IqrR1l|qvXeMLLICiAJk(tgC*+L@0t6~xv7hS)fwC=2 z+0XV)z}_h@l2ppx0kVDXc-koC;@Qeum>p$JMQ?NHHWYiee3Bns9+QaCqZ|kzOZifX z!wCEBu(A!%9*F0sLP-ypNQpq40wtpe3YDCo05)j>96Q4`4Z=w6L;>sh$bzp`0J~rU zgH-4wpR$q##5?{`gAa~Hy75**YGgj()`xKxclgz0FJV{i9lGTpr}Dg2Pi^vWnjpbLMQ+>5|$bOp#>RzdQ5OEXh*Wxln#^v z)>u?V1Oz4=5wEt}-8RaaSm7(Q#|u0c;GBhYs>UnUv~Sk$O&isJKY^ zO2rmDd0T0Fentn!O*%anmMD8R0M=eJMUYoh380}bB9tt}Ud6?49mEbAUQH<#bJ0Tx zN}=f$dQehZX@9^fKaRs&0vJl5$ig^uBDA6-4xgeRjwVys#@fJ^Kta|MM1q4r&KFlm z#4}}bti)5iOk)Lm5J*FzM+^*{FShuNqa4M?Y#0McF|fd35Oys5M<>{$1d;^&0}Poh zuSA}-Nyq@eFo=Q@;IerNP>E;pBSHKdymUo)*>S<^3A}hD2Cc~_93@wh21y%Zu85}` z6jMXsnVf4SR|?rRNKoPN2v0oFgj*>PL$rqp6m@GH5xa)bCJJ`S;8cI0ObzqGxFayx zV+09AkTM%w!f^H+hPZ@fWm=cFhwSXJriw-EI!bNmJTX|gjAY)WN~(oK0lSRkDPk8` zUVs=X3d&Y9CQFpsaO@GfpbNwav_P8eH-#)pKwMoF_tK#I9|PJP&uQ~je28sJYFqq% zotnmn`_c6fG6?VGvCn3!$b(n<<%=GrQy)l^!Hj+WtR!PK!S ztt@?xr!`jk)sBA4oJQ2qZw~7pH%Fq{p*eoS+qpUanRB0)BcP4UV;53LdCf?d#ygUk zN>IuYp$q4bqH%`hYL$Q}oTtmV0{+A0oS<(~dd8Ceol zA%@^6H%h5m110k9@fHkQFn5FQ8fb!oK{i#%3kK@AdLWnLyo{>jri>6~VHm{;_QZjT zkP`6AsU?6~SmMADGEtP0jkLLPZhaW)N5L;H9Io>$@TDdF5E)un;|=gqlyM1h9@i3- z%hpgD0gR=x^g=j~&(cSh$GJ!kuPqc<8Fr|6jgr7-SsGin z=oT`ISfHT#nEyZ7Q!oPAQYEpLL_s!7qGBiEVZa%dp}CADtoG3jyvpoLHU=3$I36gd z!5l|Mn%B%~qy>@o+L9T)?(l8Ixdgc{yB zyZ-o_RNVhPjkXSE!@+Y^CE8x71IQPU)mKSInJ$;mg0wy)p3?d%$Gaft#D}cD$|*b~^yT_B@udo>||H`uC8Y+n4o*<_nE zv9xSYPfB)HsQx}1`L`Lxk75mY0-wez_Kf396`TRG;zmU!h%(WZI)6Pq*RQo8%8U>W ze>Olt0R7nj{)}Pmf)@5S{FvqVTY@(WB;jbft%4v=3`$ovx9dYgsD zoV0cSi!muMj^^67;Dr%YHmL-a%xo$>c;G5=gcCixp(l<^(m3fnZ<~}+LRK+*$Vqv0 zUuZ8QEmTx4B#Q^dci31`Tb7N18Ah{W=?Zv&S5k5urMft2HKsYTa-6ipVK(7RIe~); z$tn7!qQOa_vL59FI!bm<>DtqrMtn5tIr>oMhwCm&On( zZ_i+X1cBwrILVh@5|TpFGZVUrgO#K|9f9*mjHjbqPuREP3?~F$#XKBbMF~a!L>{6K z{CPmYBUxr?kamHh(>p8}b9~d?5kVlr!#a>MJeDOtg@Z~U_QInhm?o462{Jnzdq~Dv z0d9n}*j9AcPl~h|FT|uK=GOl_)b{39B+Uv`t-7$79MQrxy4r$m82)E%7rQaBjD$Gd^u;oUxO4w>{68P_4io z21s;ul7d?R=Q@bUv)9+!wk_)ht4%JSO~3p*TgK8hh7wk&Ga{T=b8*|#rCy}T?#nD? zE~jH6WgkdpDQJ+2DPvkL^AfJIG4kJyv5>a~or1J+hD?W4VY43S5F<#gBs2)n5Pm?Y z(42mj8-@c2pE$bce*Ydhw!3J;DW2rdG3OBBI3!+;=D1j80G||O@6$u(@Ygw;@2Cm7 z)}FI~5CC92;1MWzLJm*@d5toI3L%zKj$HE}W#p(+V*#EKGIR<;Gi}47OUb~DUd^-o9JS)1Cbkc+<}r{x6mY~a zYEoqlQ)SAm!UiB$6Tu#QDwpPvBT{ADY%)6lU}R2|r-s-NyKS)>9u6pwqf^cK0`PxS zb6eRjCL4(x4rRBk|H3N!qUGP1h#kB@VFElNE!=t?;{ywtpe<1H*b!Us#{mzp6v4ioHE2 zzOx4qEX_(UF7~{JKC`hw*P!p!EqWGGkeLUSvrG6 zi~b;khyB8q@rKZ%jm2d&HXTn&#p%+{xp^GD7iF1ItVJ8QmGBTK`YC3?-eP%7;0+Bv z0)W#8${|pmgmjsekYeFtic7BSgt1T?o66ZHW>)&>fZCF2_5O#IZT@S+`Np=9Yul8| zK(>MYRE8%xr{IR{5DqkRHQSA~zCK+zLkUg@wVN&&7+7F)OU_V?Bn)UKA&CvE)zId@ zuB@?`p)8}4lcvtJbzav-rZ+->ESQU7f;1nRBGr*r(5rIr7A=p-eW)nC`OtEoj?@~` zRMMDGT#8|tiX2!}ku$ikCCyzqgOQUS((QvMIUW#~DYze!&T&-3AW&j%&q{7o>_|g^ z>Y@2I&-I>iCpI1CF_=4$ixmhebIJ@ll+6B|L9eXCAuoZ$3OLNF2XmSN!7#k2JI>1f zG(6E7+_Sf@b>NV==)Ux);MQQP)z_LdJgI-2HMmq_+>pp<5r3u~oh&9M^-oAj6#FNM zc_wkd$ehCUE`n{t#zwkP(UJZ`hez2OZ){R}VE@FV$f!uax-|o8*78qGi1Z&-JD_$z z&XbK6-rPrJ=3F_BTZ*45__2*#-?stRoHui=Q5$IBI5&L56B33e;M+_vb8YZ#fc8rG z&PV-o*3NOB_}Z+SItb5s+vrVJxfCFGL|ZWdKlnQ8UZC9DsooHEqfos_PwLJH4@AGkso#_5ICoaJ$(w5y zci`1a{t#H8V3)R0j_qN2qCW5lu13Bzsf z+k9n?yBRc5oDkBx;q-vw^Y;yIo1BqQGu1t{nDz{FfFjaWrJtF2iND_y227qe8bgpq)=gArMXeHn@?Bm+`HgR zqjOnjx+a$ixnI2Cx7}R%$(Q~ZkUDF)`^fr<{u_oHPJMm6QFpIr>zgjw9bRkfV_nP8 zU)*K&Y`dl%AryZf>mvu0Mkx%be3EYC;3se&>l^qn*?*KqFA&5d412W2$!{k2B3 z=s$0JUa!^lSf(kU#L`}!kISbXW!zu8pxyC_=ExS6VunP&v2;Jar_kC`8&`G|->2hq z>0!^7zqMa7ZN-6(FCYE%XtRI4{F8GZ)q8yN!m#S@gNxnVH|53gXT{GwDxLN=WJJ{3 znteiYXKq~ZQ>C}P25v1m^OXAD(K=5)OM3TR-|c&wEVA@nc)|Pawfv_|>rN}!QtRTK z+j3;an?9A#g>{|7?f%)UJ~6g(WXrSZS-$rAFQ~Q_5J>LGa6l4(IVD= zPoqCB-EoP$`iFPc>v@UC29McRxruxwww0^t`t;8I25mifIC8O}?d=lk$EEV~_wOn7 zWxd#EnXi7l`fgj+G(X#emT!%Z_LVQCpIx|c$01#pX*-t6m)@^yS8=}Zyk4c&=i;|i z^*azfa^uX4KQ2A_K4@x}xuJO_N3@Q=xL9%<_~y+1nff`!7u+1w&eUhgeg-%%R^>2UD9+`?^*w8f9rSEW$9dn7hIEm9p|bZmuUIf)jQo!@7wmpw)oqv=Z5>g z3*Gp#t@NZ@vl4ZRe^$c(zV@`dU|-^e66S;bMqlZ3r_A#oD^ELl>yhc_RjmhzaSv{P z5kBzhozjcTJlT3`N{>xi- zc;uUJ^d3~JYDQ#UeyQ1wpY7Y{x#-{u^H+1X^s4{5r&_uYFu{A<>aR~#zjau3Yi8SK z_y3qzCU?ZQXrZ^g>je$%#$eBu3GUNm%Bo$j5xSp2&-kE}EERg1HE`Jv18_bMj0T(D^9 zfr;Z2?(U0^cr-d;UXKp@Pi)t|So0>{Jm%S=S>8Q0(}nL=ZBt)r`5@psc~fZTFLzZx z`MR=St&vwU>+Z;WVNOh%sP0lBAl>Wp{&A@Zkqx!&nvLHh)U9@H!lO%>b-lyK9rMgD zA?J6mHRSu{J;TBxRQs>jzW0Z}>dw6}mf5k|tpoB$%<(tg4quelITEq|A%&-w1) z*}e<8HE}zZ9@-x@co8D4RIaB$;fTqoNtV{?wobPoy zr`wsA`J+pAu3YI}waC13Td&L-oHZc))$%F>D_3%Dy?%427UgD_3Yl2*{nKqTf4^~h zcte)~b>eooq+K-p_OOjD>tT<*elc8Hxke!aOg)$MOsi9`LGONLMucpM^LV+t=eqn? zylzb=_dy|#9$f5^wy9dyX1VHTn|FowojFnrc{whvS4t~>;;MtmwewE=@#4{UJD$&f zTs1>|_36RkUr&DV`P~|6Sv``D75HA?8+|=v$Gj}pk_}VheE$flk?iK)?$O%LX|)4f zjgNN!u_!4yu4&dP+k@{yXT{8W7klB)i%qIbuaVJg%Dt(7Gso3J|QZq=^eH5HS0v{-goYAk)WW7Q>(%F+D{*G}(< z@SNq9v#7jhn^T@|8$CGbzUQ~?wWeIJ>wmp|#a|n=OBUr9ZMCf$^)GYitfB9n@AIF% z=`(F>$??C}Sz&%$d~Z%_$l71F#eP>aX3-3{Ed{$YtIN&Zf3&LR8rP-SX-ibuZgD>~ z_$Bc4^ovJ5CRLw3snMFwU2kscf5GGC_R+ig%<21Mv-XK=X2!KDJ#j=oFH207h{Y2w zr?1>yqjsi8&yY8}Ub!qD_`Cew_csr9*?iyf^UzZLYwYg2e8t2u4fYP4_hQ-B`+0xV zeqJwo&WvR-O%hKZ(SKe3ZvQVEeb#n$?HLkvshFW*MB8#n8(&vwTji|wW?<6I zUoUK|TD?ubQYne?SDxF3JqoN)vFw~FONylq9A&(mt(Yy6KniU%E`M zk~1)|f8)~4->t6awzXLM$&DXPSbbr`>C8UGW-r(6-#V{L(t?e)9M!A>>oBj^=jROh z>PFwlpL+#&@UcaAI+t?o%5blopU!wR>wn1-)_AV(nrE^3=j9SlZ}g6Pb}1nL_HA{^ z@ai=VX5|il6Z&WJzEQ^hm8>H#EGrDS*URi}Uc0Q_(${rXlwWnF>+v{GqUO&g*@6+MOQj4d|8J8V$r~QnYLuU_oan(IN z<)(L3w|z1tST`)-^&#O*eCd_z_I7HNdM;pG_q*Lv z<&!VI3EC04c~*pHKcCurH>54ORO?Ku%3q|t6_3wtdHLeWS533ihTRTNH|E!VA{@xP zKjr+M^q*JNG&C5q_Q}O>FMZxOCVNr&OR-n?e)U6Vuk+U2k-A57?o?iR$-m6&F?UY1 z>U|(N-lJpXZzKNPd%9Qk;!9jk&K#7#=HVOL;Q@(q^`?hppRbzyxh?zFjKQv7_l(Ru z_RY5ee;jPJa@yu{eB2d_KK-{=$I}butZQHQS?AQAtxo>@+t~((yX+eE)_q)7gLzl` z3>)|B`ePo)H$QLE`FHOJv(Eq8r}e~lS=SbpD7VA3G4E`zaq_pmrI#J?(O>n;-adM` zZ&>WrI}y(gZhqUhi=k(gbLoG|WiO6w;A<{*yIS-F_bk__I_uxJ{i3;5eLuG4;+401 zwCj<4>gSFgYY)^}^nSR0Q~8vX8g~{KWcp1mfBMz((H_OhRj#!6+=_AsYTo#^!1}Dy zz2km)9vd!|no)B`iTx#RtUC0IZo`z74IlCo^D@ipu2xH5v-#NejFEjC_PKpzN%`4* z-lWJK|Fe8@{s@pYxP*J zdc9oE&_zQ-U#?o&<6O}0!j~(KuW3^2N$Uu`&Jl!zS-)+^9P!dqY-OpUr8<2Nr)eZ^fkSNrA;qwkS-mRO3eS+gEo_ zB;8(c@#$T!U8_gUk2F0D|D!bby=5ww*P|nT_0`r`VCA&qCfoE4N^EqnWb zi#vn%Or5$wJ1Qsoyu#r)}I9hkf{F z*Q4r882z2wba|3Xcv_>tu5q1j&-~--?d4{dbN_9^qC>T3x%>9*{^-w+a@oVfUw#{L z^3kkCGu~XBxNg~_3-ZK;cTCMrHu%wPY_o{;vZndm^44dMPEU(!bT4^gcB70z+m`)j z`!Cfuv}$>>)A5vgy*8D+(D~S`!l1UNTeU2)xZUUL=hiLdm;9#JyIXxdp6*_HYgF;0 z1M{Oi%B(H@^M$2Vx?I?m(DO1^rry}(qvdwj++Ai+Ov1jrfIG=Ou2*?7bWfY@&t#Qb zv-MtQlP{iIyQ@sMj-5YmzovWaq(j9YhbBMyy-Y5*NO;qD+_R{Z`OhC+vPIvoc&&Ya z%jR2SZ%x)b$~n=kOqX2`XMO$oj)YS?YDIf@DOr6KcfI?s!@Cxi_gZnW^~N$${YsR% zIP`gqhlTmmJ2$C%Zo#AySG?X{Tk~SA+{V}~V5atz?D9PErQz7RI!}81;#u!$$i7V% zJnL<{chu#~=8gBYYfgT4;dF#%DR+8izY9-FY>2dWUtM~Aq3c%lquI~I>9cnC`0Dbz zQBCjE-1qF(kg-cQJy_;{Gv?3G;V0T{UN9rHFlt`ijZ?-?UK7=1UDZ}6^Ga=98f6PC z(P&_e27wuigZ>~UnIHfKnm9V43GO0a;3#QPaji%o8D;+wE>YnEAKPcGUz{Njmq6-$j;u&nOGj=wit)@Fq;H@Ycg>IA zU$*c0{o}l)-_!#2PaQA!Tz%+hl$YV!>}l`D^*!9^_bY`hqb)8b&zrlRco^FbJahlZ z_?l^Nq}cMktSi+MF1`-^s>9NVgVs_(?xT8pZ+cj%Q`LdG5?_bk40*ZQ^HTFYhmT** zO-%SabHt_ktj=A> zc5$S6$|G}97P$sTy(=;ywLYm=bE@}1f7MUwR;PL^nqTLW24?{6J^?^=0$^*nttPTE zk8LsAn(&WpS=+`1EkMeuAG)UEhq5=0`T~0$|0e-Ekz`XNeZq0o)WiA@jpnUP+njDoP)SA>vQR zpZkyqQPBZTG80&InQ_pe4k%6-MMaxBIJGV9;52PwU5l6By7Iy6DL>C zY@9mn?97h8SM&Q`RAVnzN4v~B(_7TQ24i3PQ+1@vym4t`?$E}5QShZJ^pTC`_*?n{IEQS0F#QLPv6WshrF*#g`$? zON9%4Fk?zBX2nRmL18IqkSSYbpBu}a}FH7=>VR=k& z+4x#oJ2kv^%1hJQM1iy{kC#RlO8(7dH)g}NC9Q4FYnnkexw+|n>g{jZL%nBau69B> z>|r^)%V(F5%c*Q^>E)M?pB$=vTn_7<4D|+SNbzK-_)jmtV>GEETy=dOI*>ZYyG+fj zM%B@67R{Pa-_cEGVMBCwV-H;6%6Zid3LRQE98#n=v^xVE&CXfV)cS7o(X6UqeRnh2 z@QVozs*9&dH>d*}X})BG8lVQ70!HQb5x7r-Iv%4R;r2167Y*@;Jja-Rg6W-14|z^7 zy~EHv>rFrdLZ%L8TE{PXxE(UB<8~L*dt*FZ+}^_UZ84rL+{WyKywX-?*coHq%I%OI zZ0F8)X4)NV@8b3@ruWC%d$_%a>F>qbd%3-r>HA~tecay1^aHW>0JjGMelZw!xP6e@ z2f2MX);`4TLrg!)?IYYi9-|-O_A#axDOPmIbByUHnBK|skmp2b|Bf!+41N(Zb#Qwf z(|fob@~q=_7t?!VJYC%0!t`x1o-N$o%JiKvo~_*8-qjyqpLWNXwsU(I59lv3foB)D z_b~mv7|$MV?`8V_7|&jA?_>Ib7|%X#4={ZY?N*^frU8C&kY5~*F&(5f#RB37_6!*g zF)^)gF8GiQ!C2OEo?6#!{z!Z0AfS8~KBl+R=I{ zwx04Nt;Zx^MF>kHvEKZvE*W>6&5p$gs(^L{A4p&9Hs8N$BJzoO^Q!S72y18#*!}RT z3mo7gjD$G~KDG!iKZb&ur3YZvl(0B%K}me4a5!>ThF_8U3YWN z0veKyd3#E{m|hWAGiFd)Lbn7o2^l#r5F=zEIlu_v4!52^1^mEbwSdzd8cTg-kFQel zvN;nH|Lj#X&91_{0TQv186TfPYY zOjzw#wTK&OhC0#a!5!K1zqPv9rH`|u;K+VO4pgymM6F?UW5DqRaE+|ki!bxIu|Ry^}np? z7w8vZ>NLNTf(l$ZqQ|kO}RTZju_%5@IXlUg?#Q!=M0w)gf(i zsh@Ne#g%V64wSk`x%N^&P4=+#ep$e@2TVmTU?i*hm5tUQjmkHP!}Kg`68}Zd zteb0X*D7fDvrXa?P+%qe=SG4iAZTgXgu^B&)ADLqffz#@x_MYP>u{NH8dYUo>!e7y z-dKJ*-Z)@Rr8AoRqDJ*fzf3oxcDTqDZkbN5EckhMEC6Pg#H{1c*b43NMiv^tjP=$_ zQ`}w^fQN0^3(J<;J7ew8;+-;XJ>n~p&Sb_1Lo>3+RjOt^$UdD6HA3%1M!*+<76me; z7BgiV1gUJ14>wv%asl?I8gAPmZQIwho3c>IluTlq?Qr>^p{jiN^gKqmt z^nsFJg^7M;xfJ3Qz}f6)uYfXiM_^(SQjy@$j+3;Kr3IWZGwhs;aS0+$S} zoi8~Eb|DYZ{R%gO$FK)8Uy*hUrelvoFb}fzr^)u2lAMopbIaT$0>*aB-Tct#a!eJt z6+``!Q`#ezKnFpC5FHUYAVNxeWZ>A*RE}%a!S6kOej>e5)ce2{8a5wUE=V z3L338^z0__AReQe#CP#<+))%Q?Y`3_{sIadg-s9!k)#|fkrs)glI=aZ6fI!I05IEb zmFtwErk zo58w(hIYmWXZ+XA%Ik}wJed{3tsuc8=fEZlKp3dQD@Bu0d~`lMQqV+9(CpAqo-m79 zhbY67Wihd+4Xh<@<-?Q$mMBCIpcTObXQ6mxyiEgl=jCNCnT3yol-B!(+?Y*LRM;^J z&7-lqeP5 z2s1Mc!H9%|GyxD*g-HSj)L(oL7&TQN!^62$WOc`S?(f2b`tS5TbzR!A-|HlFFq^(} za!%gMBTwt~py*1yn_eJy0!7MnQRxHB2pGW`T^C(IHV#nO6JTOmA0j+FP8>CT2XFmW zLOOj7cU&dC!$X7gP77kW`Ru=!TR_-H)k7179#m#kZ>2eZ<@x5d-mw8LH+m!i8x?=Z z(+MahIggMup1?D8;n8U2~M2sbCG0c0|)*9x`Y6P{QU$`flreTwkx$7}Xgq#sD=00MIf{(Q)F4hyf{4(;mk*7{o z9eeUs8r!kder9@}q9(#8R5PH+T}Zki7O?SokxK#o;Q;!*5z&ll{7mBc4-Mxa^6OYU zQy^z3#&CuSLyaS@4KgAY5bcc?YD9!)ljxyms!437XU4T#;p`|Tn#3j$$lqYVAqORA zg^jJYli38eH={G{UNV5*z^^1uxx)t*se5F>xtOXSGERTyz zJU2<4Q`jhaFl)~Fg%tVT_u3tCDKIEqp-a#7rc9Y2FE=E-1~g`9|H%1z)nj79>qWR!w4 zbWjg7aE3B)kg0Z4NWm734O4K6Q!q46Iyk0yd!E%nyPIu(7nOZ{J@5Oz=@(U zxW2x;SQp?f*-hZ80Z%Fi0KKQ&O5mi6rRi2ey0{&nTM1YU)AF(q6!%puTXWdJs<0Bh z#zlvZjYv|PZ%bOK9j;Ed7D-(@HN=HN2h zw_vL9%1MP+K+rh40vdVcWD(4dymG=>k*lNcrVsQ%4h5l6?vS;wA)la22Z;$x5?gbsOEV&EYyOMI&%I^x10KqO&@#mN~Pq zV;bGlZ0J}U-R$#M1vBw+oyj5aX#{SCmOuW-vrehkzQSpSeP%?xw{gvsEMNJASBR!@ z&4K_$q{_aUvTT%JlFF}MTn$;15Ej@z6&@WmU<3)d3|@phNyya$=xxC+Xo+KmMa7h+ zBlfl@n*o<}Q$W2Q?zQQJm*nIg9!?1KP&MTQ|1nQ&&jy)j5WPaF74HHb<6RUc@Q0C0 ztwkh>20^h-ti3p9MtKx|&?KH>=N}ShkWQflUQh$4&;)w5C?2TdDP6CS{9jTEf@nUv zvns$VSYbrzhnT_wgFrhzK@R5%oH0?!;r9u}MI2BHolZr^&N_%j>(?UXd)Nlt&~W15 z6DSggMF|9n1NT0hFD%AwDL43V}!uXJBMm_1))`_EAN}#Lhv2k*^mWd zR62$7=`(`wuoJ;|GzZ`t``WM&_>Jqj5`JUzBmB08@f*WS_)XG;f5Y`GuE7Ax5rm^f z5RL~yI3APC8_&&_W#;qe<^(Z#rEBB}UhPxi6#|ZgS4%&8hVc3okrJMaD(R>+5`Km0 zTysSUdXaGA&>Mb^`)JLDi0iVo)g|a<7>%Hpd=b!V0ebb?81#lim#@9MVt}C+*@l81 z;5+Tby38>68n5$B`oj6^&M*6H$5kt8;@D?96eI*^{u6bs4g4gGreVfGPcAsc(L~?zwT%*)WAL*8}VN zh2H%HWxpRSd|Fj+qb9DTu(k`CI-@S`887Nn8#h?;8>Xv!0e(O3UMIiNchC20q`i%g zl6`;q1FO_FFZ5iaI-WL1d&bivyJ@Q3@pSV{ku#rd8bg#n+0+zL(k04{O;^y1PdD+1 z^yXRNdr^9S+h$GgpWj@k9)G%TVDoQe-7GpoQrWJG5)Bmcs;zYJRVc}#yMrxsJyBy4 z&mINH|Nhzgayd#jIK)ylZGQEmx9C&thOKuv$XfVhOrNCblQA|p`TW(<_gm~q={xc* z&ug{s8T0J(-#jZaq|*Da)?TvYakivMz2FhgMK9d^B|M!}G1r*QFU$@8RmlWxjkDBO z5!z!Vf!z6@sd>oO84utQ9xCCqaNG6&B$oRLdMsP&;~VW4s5uUSgyZ$fX(F^SOhJ{@ zA+UrDL$AtJGv@eW`^O2I-1Y*&CoI4yCTzoeG!;OrG*z0XN_^Q=8BA1s=ijC+_Yzup zR0Ct~gf7FxE>cZ+a z@u!=c_=Xr42~^S%LqTU;vDB%6u@p-ANeB`1!Czhd^-;Xs47~VU{zE_~q(hRkoO-kR zCzn*dy(&h5li|X^Jn@sd{Qt!1#@c==W!CmxGH-sj{gDd}4}}qK1&cTm#ryD+_qcEN zymZ~&o*g$SXM*|tj=wXX@4P{6d(td;#Zx^`nzmQomn~-8uE%m^fRr6; zA<=ASwt0S+e}R{+z!?VsFrsJ)p?DaH%@2PzcGec$Ur~Jxe+%pDkdOlb&;#eu`0`bS z$986*vB$6#m@SRbJv#hun#DPsEq^p8{MA36{oI@6(| zW$t^`pNWC#dDW<(FHmr6$^zjBk{T?HZZ2d+&yy}zArS$yB(P-$U&UPkH2oSam%X;^ zOxQku?LIsFj%DuKeZ!fIIK2CMdA^zPdJTRTy*{RCE6Q&;I8IAwK^>G=*M#?K8h6LN z-OZu)HEIjWVPc~vpkiCM+4=gp0T}oW)0rpmIc=FBgXrK|1QJZfM<5+00}fOAHy>nI zjk3ncgr8t7$MR$r<0K7oE!sa6mhrJesE3Y0Bxv{asa925y2-9vE#x0r95-#nLm`Sv$&K`r0^=2&yXo{8C5 z?3iGWziD~g|7!8y+?duvL~T#>D!Z3sWvy3%REXU@w-b`)6`mS2f< zkJ`81^uIMeSDqcu!V0}@t@)3)=GRd*hnbaAjsg0}TEyo>U8FP%TY&Q0XF!>(*HO@1 zYTDi&bJ?{bn-h93T_jgKvqcvE@*#D8DX3RQvkB+Qq1GJYfE+J<&%FJ1_3W#8H1->p zCEW;q@JHL5Ld%&XvT2sv5ja=?r$sjVn^8F4lbqT8qs)|_-`aQiU!Sk~((hEsF{`8_ z<_RJ0%nDJCXKQ==J?&=i&%c>OXR0z|_xb@^;zi|%8}Y(4}=PAVn- z7W7BQ`g<`x6YK9pe{8J(JM?Qt_U{tYM-oOX@G%CAi@jfh1@$TYxWISfy%&35ErqDy zzB&8dsiRAB-va$Zxy`rVZMm&PekdWt#j(+=(XWp6_n@DT^~YnH`1{GH^j`#jD82c= zcdt_y+I`pk;!mX%=AFNtVu}Z*%D}vGU^ad~JMd8BKVq*IV)73BPb!uA8>7C^`%w#L Z6PQzo+QjljcQ2N{yuu*qo4p^*`hWPz0t5g6 diff --git a/static/js/workers/deltaWorker.js b/static/js/workers/deltaWorker.js new file mode 100644 index 00000000..3a86f650 --- /dev/null +++ b/static/js/workers/deltaWorker.js @@ -0,0 +1,330 @@ +/** + * OxiCloud — delta-upload worker ("upload only what changed"). + * + * Runs the whole client side of the delta protocol off the main thread: + * + * read 8 MiB slices ─► FastCDC chunk + BLAKE3 (WASM, same crate and + * parameters as the server) ─► negotiate hash batches ─► upload only + * the missing chunks (framed, bounded concurrency) ─► commit. + * + * The stages OVERLAP: negotiation of batch N and uploads of its missing + * chunks run while batch N+1 is still being hashed, so wall-clock time + * approaches max(hash time, upload time) instead of their sum. RAM stays + * flat: chunk bytes are re-sliced from the File at upload time, never + * hoarded. + * + * Protocol with the spawner: + * in : { file: File, folderId: string, name: string, csrfToken: string } + * out : { type: 'progress', hashedBytes, reusedBytes, uploadedBytes, totalBytes } + * { type: 'done', status, body } — conclusive HTTP outcome + * { type: 'fallback', reason } — do a plain byte upload + */ + +// Absolute URLs on purpose: vendors/workers are served verbatim in both +// dev and the release IIFE bundle (same pattern as the pdf.js loader). +const WASM_GLUE_URL = '/js/vendors/hash-wasm/oxicloud_hash_wasm.js'; + +/** File read granularity — large enough to amortize Blob→ArrayBuffer. */ +const SLICE_BYTES = 8 * 1024 * 1024; +/** Negotiate after this many freshly hashed chunks (~64 MiB of content). */ +const NEGOTIATE_BATCH = 256; +/** Group missing chunks into PUT bodies of at most this many bytes. */ +const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024; +/** Concurrent chunk-PUT requests. */ +const UPLOAD_CONCURRENCY = 2; +/** Re-commit attempts when the server answers 409 still_missing. */ +const COMMIT_RETRIES = 2; + +/** + * Typed view of the dedicated-worker global scope (jsconfig targets the + * DOM lib, where `self` is a Window — cast to what this worker uses). + * @type {{ onmessage: ((event: MessageEvent) => void) | null, + * postMessage: (message: unknown) => void }} + */ +const workerScope = /** @type {any} */ (self); + +/** + * One chunk occurrence, in file order. + * @typedef {{ h: string, s: number, offset: number }} WorkerChunk + */ + +/** @returns {Promise} the initialized WASM module */ +async function loadWasm() { + const mod = await import(WASM_GLUE_URL); + await mod.default(); + return mod; +} + +workerScope.onmessage = async (event) => { + const { file, folderId, name, csrfToken } = /** @type {{ file: File, folderId: string, name: string, csrfToken: string }} */ (event.data); + + /** @param {string} reason */ + const fallback = (reason) => workerScope.postMessage({ type: 'fallback', reason }); + + /** @type {Record} */ + const mutHeaders = { 'Content-Type': 'application/json' }; + if (csrfToken) mutHeaders['X-CSRF-Token'] = csrfToken; + + let wasm; + try { + wasm = await loadWasm(); + } catch (err) { + fallback(`wasm unavailable: ${err instanceof Error ? err.message : String(err)}`); + return; + } + + // ── Shared pipeline state ───────────────────────────────────── + /** @type {WorkerChunk[]} */ + const chunks = []; // every occurrence, in file order + /** @type {Set} */ + const seenForNegotiate = new Set(); // distinct hashes already sent to negotiate + let reusedBytes = 0; + let uploadedBytes = 0; + let hashedBytes = 0; + let failed = /** @type {string | null} */ (null); + + let lastProgress = 0; + const progress = (force = false) => { + const now = Date.now(); + if (!force && now - lastProgress < 150) return; + lastProgress = now; + workerScope.postMessage({ + type: 'progress', + hashedBytes, + reusedBytes, + uploadedBytes, + totalBytes: file.size + }); + }; + + // ── Upload stage: bounded-concurrency drain of uploadByHash ── + /** @type {WorkerChunk[]} */ + const uploadQueue = []; + /** @type {Promise[]} */ + const uploadWorkers = []; + let uploadsClosed = false; + /** @type {(() => void) | null} */ + let wakeUploader = null; + const signalUploaders = () => { + if (wakeUploader) { + const w = wakeUploader; + wakeUploader = null; + w(); + } + }; + + /** Encode a batch of chunks as [u32 BE len][bytes] frames. */ + const encodeFrames = async (/** @type {WorkerChunk[]} */ batch) => { + const total = batch.reduce((n, c) => n + 4 + c.s, 0); + const wire = new Uint8Array(total); + const view = new DataView(wire.buffer); + let at = 0; + for (const c of batch) { + // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM + const bytes = new Uint8Array(await file.slice(c.offset, c.offset + c.s).arrayBuffer()); + view.setUint32(at, c.s, false); + wire.set(bytes, at + 4); + at += 4 + c.s; + } + return wire; + }; + + const uploadLoop = async () => { + while (!failed) { + // Take up to UPLOAD_BATCH_BYTES from the queue. + /** @type {WorkerChunk[]} */ + const batch = []; + let bytes = 0; + while (uploadQueue.length > 0 && bytes < UPLOAD_BATCH_BYTES) { + const c = /** @type {WorkerChunk} */ (uploadQueue.shift()); + batch.push(c); + bytes += c.s; + } + if (batch.length === 0) { + if (uploadsClosed) return; + // eslint-disable-next-line no-await-in-loop -- queue wait + await new Promise((resolve) => { + wakeUploader = /** @type {() => void} */ (resolve); + }); + continue; + } + try { + // eslint-disable-next-line no-await-in-loop -- bounded by pool size + const wire = await encodeFrames(batch); + // eslint-disable-next-line no-await-in-loop -- bounded by pool size + const response = await fetch('/api/files/delta/chunks', { + method: 'PUT', + headers: { + 'Content-Type': 'application/octet-stream', + ...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {}) + }, + body: wire + }); + if (!response.ok) { + failed = `chunk PUT failed (HTTP ${response.status})`; + return; + } + for (const c of batch) uploadedBytes += c.s; + progress(); + } catch (err) { + failed = `chunk PUT failed: ${err instanceof Error ? err.message : String(err)}`; + return; + } + } + }; + for (let i = 0; i < UPLOAD_CONCURRENCY; i++) uploadWorkers.push(uploadLoop()); + + // ── Negotiate stage ─────────────────────────────────────────── + /** @type {Promise[]} */ + const negotiations = []; + const negotiate = (/** @type {WorkerChunk[]} */ fresh) => { + if (fresh.length === 0 || failed) return; + negotiations.push( + (async () => { + try { + const response = await fetch('/api/files/delta/negotiate', { + method: 'POST', + headers: mutHeaders, + body: JSON.stringify({ chunks: fresh.map(({ h, s }) => ({ h, s })) }) + }); + if (!response.ok) { + failed = failed || `negotiate failed (HTTP ${response.status})`; + return; + } + const missing = new Set(/** @type {{missing: string[]}} */ (await response.json()).missing); + for (const c of fresh) { + if (missing.has(c.h)) { + uploadQueue.push(c); + } else { + reusedBytes += c.s; + } + } + signalUploaders(); + progress(); + } catch (err) { + failed = failed || `negotiate failed: ${err instanceof Error ? err.message : String(err)}`; + } + })() + ); + }; + + // ── Chunking stage (drives the other two) ──────────────────── + try { + const chunker = new wasm.DeltaChunker(); + /** @type {WorkerChunk[]} */ + let freshBatch = []; + let offset = 0; + + /** @param {[string, number][]} emitted */ + const onChunks = (emitted) => { + for (const [h, s] of emitted) { + /** @type {WorkerChunk} */ + const chunk = { h, s, offset }; + offset += s; + chunks.push(chunk); + if (seenForNegotiate.has(h)) { + // Repeated content inside the same file: the first + // occurrence decides upload vs reuse; later ones are + // pure reuse for accounting. + reusedBytes += s; + } else { + seenForNegotiate.add(h); + freshBatch.push(chunk); + if (freshBatch.length >= NEGOTIATE_BATCH) { + negotiate(freshBatch); + freshBatch = []; + } + } + } + }; + + for (let read = 0; read < file.size && !failed; read += SLICE_BYTES) { + const end = Math.min(read + SLICE_BYTES, file.size); + // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM + const slice = new Uint8Array(await file.slice(read, end).arrayBuffer()); + onChunks(JSON.parse(chunker.update(slice))); + hashedBytes = end; + progress(); + } + const fin = JSON.parse(chunker.finish()); + chunker.free(); + onChunks(fin.chunks); + negotiate(freshBatch); + const fileHash = /** @type {string} */ (fin.file_hash); + hashedBytes = file.size; + progress(true); + + // ── Drain: negotiations → uploads → commit ─────────────── + await Promise.all(negotiations); + uploadsClosed = true; + signalUploaders(); + await Promise.all(uploadWorkers); + if (failed) { + fallback(failed); + return; + } + + const commitBody = { + file_hash: fileHash, + chunks: chunks.map(({ h, s }) => ({ h, s })), + name, + folder_id: folderId + }; + for (let attempt = 0; ; attempt++) { + // eslint-disable-next-line no-await-in-loop -- retry loop + const response = await fetch('/api/files/delta/commit', { + method: 'POST', + headers: mutHeaders, + body: JSON.stringify(commitBody) + }); + /** @type {any} */ + let body = null; + try { + // eslint-disable-next-line no-await-in-loop -- retry loop + body = await response.json(); + } catch (_) {} + + const stillMissing = response.status === 409 && Array.isArray(body?.still_missing); + if (stillMissing && attempt < COMMIT_RETRIES) { + // GC race or a chunk we wrongly assumed claimable: upload + // exactly what the server names and try again. + const byHash = new Map(chunks.map((c) => [c.h, c])); + /** @type {WorkerChunk[]} */ + const retry = []; + for (const h of body.still_missing) { + const c = byHash.get(h); + if (!c) { + fallback('server requested an unknown chunk'); + return; + } + retry.push(c); + } + const wire = await encodeFrames(retry); + // eslint-disable-next-line no-await-in-loop -- retry loop + const put = await fetch('/api/files/delta/chunks', { + method: 'PUT', + headers: { + 'Content-Type': 'application/octet-stream', + ...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {}) + }, + body: wire + }); + if (!put.ok) { + fallback(`retry chunk PUT failed (HTTP ${put.status})`); + return; + } + for (const c of retry) uploadedBytes += c.s; + progress(true); + continue; + } + + // Conclusive: 201 created, or a real error (quota, name + // conflict, validation). The spawner maps it to the uploaders' + // UploadAnswer contract. + workerScope.postMessage({ type: 'done', status: response.status, body }); + return; + } + } catch (err) { + fallback(err instanceof Error ? err.message : String(err)); + } +}; diff --git a/static/js/workers/hashWorker.js b/static/js/workers/hashWorker.js deleted file mode 100644 index d2061ecd..00000000 --- a/static/js/workers/hashWorker.js +++ /dev/null @@ -1,79 +0,0 @@ -/** - * OxiCloud — BLAKE3 hashing worker (instant-upload support). - * - * Hashes a File off the main thread, reading it in fixed-size slices so - * RAM stays constant regardless of file size. The WASM module is compiled - * from the exact same `blake3` crate the server uses, so the digest - * computed here equals the server's content address bit for bit. - * - * Protocol: receives `{ file: File }`, answers - * `{ ok: true, hash: string }` or `{ ok: false, error: string }`. - * The spawner terminates the worker after one file. - */ - -// Absolute URL on purpose: vendors are served verbatim at /js/vendors/ in -// both dev and release mode (the release IIFE bundle would break a -// relative import) — same pattern as the pdf.js loader in thumbnail.js. -const WASM_GLUE_URL = '/js/vendors/hash-wasm/oxicloud_hash_wasm.js'; - -/** - * 8 MiB slices — large enough to amortize the per-slice Blob→ArrayBuffer - * round-trip, small enough that peak worker RAM stays flat for any size. - */ -const SLICE_BYTES = 8 * 1024 * 1024; - -/** - * Typed view of the dedicated-worker global scope. The project's - * jsconfig targets the DOM lib, where `self` is a Window — cast to the - * two members this worker actually uses. - * @type {{ onmessage: ((event: MessageEvent) => void) | null, - * postMessage: (message: unknown) => void }} - */ -const workerScope = /** @type {any} */ (self); - -/** - * Memoized WASM module (in-flight or settled), `default()` already run. - * Reset on failure so a later message can retry a transient load error. - * @type {Promise | null} - */ -let _wasmPromise = null; - -/** @returns {Promise} */ -function getWasm() { - if (!_wasmPromise) { - _wasmPromise = import(WASM_GLUE_URL) - .then(async (mod) => { - await mod.default(); - return mod; - }) - .catch((err) => { - _wasmPromise = null; - throw err; - }); - } - return _wasmPromise; -} - -workerScope.onmessage = async (event) => { - const file = /** @type {{ file: File }} */ (event.data).file; - try { - const wasm = await getWasm(); - const hasher = new wasm.Blake3Hasher(); - try { - for (let offset = 0; offset < file.size; offset += SLICE_BYTES) { - const end = Math.min(offset + SLICE_BYTES, file.size); - // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM - const buffer = await file.slice(offset, end).arrayBuffer(); - hasher.update(new Uint8Array(buffer)); - } - workerScope.postMessage({ ok: true, hash: hasher.finalizeHex() }); - } finally { - hasher.free(); - } - } catch (err) { - workerScope.postMessage({ - ok: false, - error: err instanceof Error ? err.message : String(err) - }); - } -}; diff --git a/wasm/oxicloud-hash/Cargo.lock b/wasm/oxicloud-hash/Cargo.lock index b053178b..5675d509 100644 --- a/wasm/oxicloud-hash/Cargo.lock +++ b/wasm/oxicloud-hash/Cargo.lock @@ -65,6 +65,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fastcdc" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af40d8a8dadb92dc178569a5f5edb5f3056e98255c2de48ab5d59a52892e0c" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -88,6 +94,7 @@ name = "oxicloud-hash-wasm" version = "0.1.0" dependencies = [ "blake3", + "fastcdc", "wasm-bindgen", ] diff --git a/wasm/oxicloud-hash/Cargo.toml b/wasm/oxicloud-hash/Cargo.toml index 0c54d4d1..b95fd253 100644 --- a/wasm/oxicloud-hash/Cargo.toml +++ b/wasm/oxicloud-hash/Cargo.toml @@ -20,6 +20,10 @@ crate-type = ["cdylib"] # evergreen browser since 2023 supports it, and the frontend falls back # to a plain byte upload when instantiation fails. blake3 = { version = "1.8.4", default-features = false, features = ["wasm32_simd"] } +# Same crate AND parameters as the server's CDC dedup engine — chunk +# boundaries computed in the browser must equal the server's bit for bit, +# or cross-version dedup between byte uploads and delta uploads collapses. +fastcdc = "4.0.0" wasm-bindgen = "0.2" [profile.release] diff --git a/wasm/oxicloud-hash/src/lib.rs b/wasm/oxicloud-hash/src/lib.rs index 91cb8d55..627c379b 100644 --- a/wasm/oxicloud-hash/src/lib.rs +++ b/wasm/oxicloud-hash/src/lib.rs @@ -64,6 +64,137 @@ pub fn blake3_hex(data: &[u8]) -> String { blake3::hash(data).to_hex().to_string() } +// ── Delta-upload chunker ───────────────────────────────────────────────────── + +/// CDC parameters — MUST mirror `dedup_service.rs` on the server +/// (`CDC_MIN_CHUNK` / `CDC_AVG_CHUNK` / `CDC_MAX_CHUNK`). Identical +/// parameters + identical crate ⇒ identical boundaries, which is what +/// makes a chunk hashed in the browser deduplicate against a chunk the +/// server cut from a byte upload. +const CDC_MIN_CHUNK: usize = 65_536; +const CDC_AVG_CHUNK: usize = 262_144; +const CDC_MAX_CHUNK: usize = 1_048_576; + +/// Incremental FastCDC chunker + whole-file BLAKE3, for the delta-upload +/// worker. Feed the file in slices; every call returns the chunks that +/// became FINAL; `finish()` flushes the tail and returns the file hash. +/// +/// ```js +/// const c = new DeltaChunker(); +/// for (const slice of slices) { +/// for (const [h, s] of JSON.parse(c.update(bytes))) { … } +/// } +/// const { chunks, file_hash } = JSON.parse(c.finish()); +/// ``` +/// +/// Correctness of the incremental split: FastCDC decides each cut by +/// scanning at most `CDC_MAX_CHUNK` bytes from the chunk's start. When +/// the chunker runs over the buffered prefix of a longer file, every +/// produced chunk except the LAST ended on a content/max-size condition +/// — its decision window was fully available, so the full-file chunker +/// makes the same cut. Only the last chunk (cut by "end of buffer") is +/// provisional: it stays buffered and is re-examined when more bytes +/// arrive. By induction the emitted boundaries equal a single FastCDC +/// pass over the whole file — the mirror test below proves it. +#[wasm_bindgen] +pub struct DeltaChunker { + /// Provisional tail: bytes after the last FINAL cut. + buf: Vec, + file_hasher: blake3::Hasher, + total: u64, +} + +/// Append one `["",len]` item to a hand-rolled JSON array — hashes +/// are hex and sizes are integers, so manual JSON is unambiguous and +/// keeps a serde dependency out of the wasm binary. +fn push_chunk_json(out: &mut String, hash: &str, len: usize) { + if !out.ends_with('[') { + out.push(','); + } + out.push_str("[\""); + out.push_str(hash); + out.push_str("\","); + out.push_str(&len.to_string()); + out.push(']'); +} + +#[wasm_bindgen] +impl DeltaChunker { + /// Create a chunker with the server's CDC parameters. + #[wasm_bindgen(constructor)] + pub fn new() -> DeltaChunker { + DeltaChunker { + buf: Vec::with_capacity(2 * CDC_MAX_CHUNK), + file_hasher: blake3::Hasher::new(), + total: 0, + } + } + + /// Feed one slice. Returns a JSON array of the chunks that became + /// final: `[["", size], …]` (possibly empty). + pub fn update(&mut self, data: &[u8]) -> String { + self.file_hasher.update(data); + self.total += data.len() as u64; + self.buf.extend_from_slice(data); + + let mut out = String::from("["); + let mut consumed = 0usize; + { + let chunks: Vec = fastcdc::v2020::FastCDC::new( + &self.buf, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ) + .collect(); + // Every chunk but the last ended on a content/max condition → + // final. The last one ended because the buffer did → keep it. + for chunk in chunks.iter().take(chunks.len().saturating_sub(1)) { + let bytes = &self.buf[chunk.offset..chunk.offset + chunk.length]; + push_chunk_json( + &mut out, + &blake3::hash(bytes).to_hex().to_string(), + chunk.length, + ); + consumed = chunk.offset + chunk.length; + } + } + if consumed > 0 { + self.buf.drain(..consumed); + } + out.push(']'); + out + } + + /// Flush the provisional tail and return + /// `{"chunks":[["",size]…],"file_hash":"","total":N}`. + /// `chunks` holds at most one entry (the tail); an empty file has none + /// and its `file_hash` is BLAKE3 of the empty input. + pub fn finish(&mut self) -> String { + let mut out = String::from("{\"chunks\":["); + if !self.buf.is_empty() { + push_chunk_json( + &mut out, + &blake3::hash(&self.buf).to_hex().to_string(), + self.buf.len(), + ); + self.buf.clear(); + } + out.push_str("],\"file_hash\":\""); + out.push_str(&self.file_hasher.finalize().to_hex().to_string()); + out.push_str("\",\"total\":"); + out.push_str(&self.total.to_string()); + out.push('}'); + out + } +} + +impl Default for DeltaChunker { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod tests { use super::*; @@ -95,4 +226,94 @@ mod tests { "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" ); } + + // ── DeltaChunker mirror test ───────────────────────────────── + // + // The client-side twin of the server's + // `test_stream_chunking_matches_slice_chunking`: incremental chunking + // with adversarial slice sizes must produce exactly the boundaries of + // one FastCDC pass over the whole buffer — the property cross-version + // dedup between byte uploads and delta uploads hangs on. + + fn run_chunker(data: &[u8], slice: usize) -> (Vec<(String, usize)>, String) { + let mut chunker = DeltaChunker::new(); + let mut chunks: Vec<(String, usize)> = Vec::new(); + let mut parse = |json: &str, into: &mut Vec<(String, usize)>| { + // items look like ["",N] — split on '[' groups. + for item in json.split("[\"").skip(1) { + let hash = &item[..64]; + let size: usize = item[66..item.find(']').unwrap()].parse().unwrap(); + into.push((hash.to_string(), size)); + } + }; + for piece in data.chunks(slice.max(1)) { + let emitted = chunker.update(piece); + parse(&emitted, &mut chunks); + } + let fin = chunker.finish(); + let tail_json = &fin[fin.find('[').unwrap()..=fin.find(']').unwrap()]; + parse(tail_json, &mut chunks); + let file_hash = fin.split("\"file_hash\":\"").nth(1).unwrap()[..64].to_string(); + (chunks, file_hash) + } + + #[test] + fn incremental_chunking_matches_single_pass() { + // 4 MiB of xorshift noise — genuinely content-defined cut points + // (a byte-periodic generator would only ever hit max-size cuts). + let mut state: u64 = 0x243F_6A88_85A3_08D3; + let mut data = Vec::with_capacity(4 * 1024 * 1024); + while data.len() < 4 * 1024 * 1024 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + data.extend_from_slice(&state.to_le_bytes()); + } + + let reference: Vec<(String, usize)> = + fastcdc::v2020::FastCDC::new(&data, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK) + .map(|c| { + ( + blake3::hash(&data[c.offset..c.offset + c.length]) + .to_hex() + .to_string(), + c.length, + ) + }) + .collect(); + assert!(reference.len() > 4, "test data must span several chunks"); + + // Slice sizes chosen to stress every refill path: tiny (7 B), + // typical worker slice (8 MiB > file), page-ish, and exactly the + // CDC max so provisional tails land on boundaries. + for slice in [7usize, 4096, CDC_MAX_CHUNK, 8 * 1024 * 1024] { + let (chunks, file_hash) = run_chunker(&data, slice); + assert_eq!( + chunks, reference, + "boundaries must not depend on slicing (slice={slice})" + ); + assert_eq!( + file_hash, + blake3_hex(&data), + "file hash must match one-shot BLAKE3 (slice={slice})" + ); + } + } + + #[test] + fn delta_chunker_empty_and_tiny_inputs() { + let (chunks, file_hash) = run_chunker(b"", 1024); + assert!(chunks.is_empty()); + assert_eq!( + file_hash, + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" + ); + + let tiny = b"below the CDC minimum"; + let (chunks, file_hash) = run_chunker(tiny, 4); + assert_eq!(chunks.len(), 1, "tiny input is one (tail) chunk"); + assert_eq!(chunks[0].1, tiny.len()); + assert_eq!(chunks[0].0, blake3_hex(tiny)); + assert_eq!(file_hash, blake3_hex(tiny)); + } }