Delta-upload client: FastCDC in WASM + overlapped worker pipeline

Phase 2 — the client side of "upload only what changed", closing the
delta-sync plan.

WASM (wasm/oxicloud-hash): DeltaChunker adds incremental FastCDC with
the server's exact crate and parameters (64K/256K/1M) next to the BLAKE3
hasher. The incremental split is provably identical to a single pass:
every chunk except the last ends on a content/max-size condition whose
decision window was fully buffered, so only the tail is provisional and
re-examined as slices arrive. A mirror test — the client twin of the
server's stream≡slice test — chunks 4 MiB of xorshift noise with
adversarial slice sizes (7 B … 8 MiB) and requires boundary-for-boundary
equality with one FastCDC pass. Vendored artifacts rebuilt (55 KB wasm).

Worker (static/js/workers/deltaWorker.js): the full protocol off the
main thread with OVERLAPPED stages — 8 MiB file slices feed the chunker
while earlier batches (256 hashes) negotiate and their missing chunks
upload through a 2-deep PUT pool (≤8 MiB framed bodies, bytes re-sliced
from the File at send time, never hoarded). Commit handles 409
still_missing by uploading exactly the named hashes and retrying.

Orchestrator (features/files/deltaUpload.js): threshold (8 MiB),
worker lifecycle + size-scaled timeout, progress relay to the upload
bell, conclusive-outcome mapping (201/200, 507 quota, 409 name
conflict) and silent fallback to the byte upload for everything else.
Wired into uploadFiles and uploadFolderEntries, which now surface one
batch summary of the bytes dedup saved. This subsumes the whole-file
instant-upload module — a fully-known file negotiates to nothing
missing and the commit short-circuits on possession — so
instantUpload.js and hashWorker.js are removed (the /api/dedup/check
and /api/files/by-hash endpoints remain for API clients).

Verified end-to-end against PostgreSQL 16 — the cross-boundary proof
the whole design hangs on, in both directions: a 24 MB file byte-
uploaded (server-side CDC) then edited and delta-negotiated with
WASM-computed chunks reported missing 1/74 (boundaries bit-identical),
synced with 344 KB on the wire vs 24 MB (98.6% saved) and downloaded
byte-identical; inversely, a file created via delta then byte-uploaded
as identical content produced a server-side manifest DEDUP HIT with the
same content_hash. Insertion at the head of the file (the adversarial
CDC case) still negotiated missing 1/74. Chunk+hash throughput ≈275 MB/s
in V8 with SIMD128.

https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS

static/js/features/files/deltaUpload.js

@@ -0,0 +1,160 @@
+/**
+ * OxiCloud - Delta upload ("upload only what changed").
+ *
+ * Main-thread orchestrator for `workers/deltaWorker.js`, which runs the
+ * whole client side of the delta protocol off the UI thread: FastCDC
+ * chunking + BLAKE3 (the same WASM crate and parameters as the server,
+ * so boundaries match bit for bit), per-batch negotiation, upload of
+ * only the missing chunks, and the commit.
+ *
+ * This SUBSUMES the previous whole-file instant upload: a fully known
+ * file negotiates to "nothing missing" and the commit short-circuits on
+ * possession of the file hash — same zero-byte outcome, one pipeline.
+ *
+ * Performance posture:
+ * - Stages overlap inside the worker (hash ‖ negotiate ‖ upload), so
+ *   wall-clock approaches max(hash, upload) instead of their sum.
+ * - RAM stays flat: 8 MiB read slices; chunk bytes are re-sliced from
+ *   the File at upload time, never hoarded.
+ * - Files below {@link DELTA_UPLOAD_MIN_SIZE} skip the pipeline: the
+ *   round-trips cost more than the bytes.
+ * - Any failure falls back silently to the normal byte upload — delta
+ *   is an optimization, never a gate.
+ */
+
+import { getCsrfToken } from '../../core/csrf.js';
+
+/**
+ * Files smaller than this upload normally: hashing + negotiation
+ * round-trips outweigh the transfer.
+ */
+export const DELTA_UPLOAD_MIN_SIZE = 8 * 1024 * 1024;
+
+// Absolute URL on purpose — works in dev and in the release IIFE bundle
+// (same pattern as the pdf.js loader in thumbnail.js).
+const DELTA_WORKER_URL = '/js/workers/deltaWorker.js';
+
+/** Budget: 120 s base + 90 s per GB (hashing + uploading the delta). */
+const DELTA_TIMEOUT_BASE_MS = 120000;
+const DELTA_TIMEOUT_PER_GB_MS = 90000;
+
+/**
+ * `false` once the environment proved unable to run the worker/WASM —
+ * later files skip straight to the byte upload. `null` = not yet known.
+ * @type {boolean | null}
+ */
+let _deltaUploadUsable = null;
+
+/**
+ * Result contract shared with the uploaders' `UploadAnswer`, plus the
+ * bandwidth accounting the UI surfaces.
+ * @typedef {Object} DeltaUploadAnswer
+ * @property {boolean} ok
+ * @property {any} [data] FileDto on success
+ * @property {string} [errorMsg]
+ * @property {boolean} [isQuotaError]
+ * @property {number} [savedBytes] bytes NOT transferred thanks to dedup
+ */
+
+/**
+ * Try to upload `file` through the delta protocol.
+ *
+ * Resolves `null` whenever the plain byte upload should proceed (file too
+ * small, environment unusable, any transport/protocol failure). Resolves
+ * a {@link DeltaUploadAnswer} when the outcome is conclusive — success,
+ * quota exceeded, or a name conflict a byte upload would also hit.
+ *
+ * @param {File} file
+ * @param {string | null | undefined} folderId
+ * @param {(pct: number) => void} [onProgress] 0-99 while transferring
+ * @returns {Promise<DeltaUploadAnswer | null>}
+ */
+export function tryDeltaUpload(file, folderId, onProgress) {
+    if (!folderId || file.size < DELTA_UPLOAD_MIN_SIZE || _deltaUploadUsable === false || typeof Worker === 'undefined') {
+        return Promise.resolve(null);
+    }
+
+    return new Promise((resolve) => {
+        /** @type {Worker} */
+        let worker;
+        try {
+            worker = new Worker(DELTA_WORKER_URL, { type: 'module' });
+        } catch (_) {
+            _deltaUploadUsable = false;
+            resolve(null);
+            return;
+        }
+
+        const sizeGB = file.size / (1024 * 1024 * 1024);
+        const timeoutMs = DELTA_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * DELTA_TIMEOUT_PER_GB_MS;
+
+        let savedBytes = 0;
+
+        /** @param {DeltaUploadAnswer | null} answer */
+        const settle = (answer) => {
+            clearTimeout(timer);
+            worker.terminate();
+            resolve(answer);
+        };
+        const timer = setTimeout(() => settle(null), timeoutMs);
+
+        worker.onmessage = (event) => {
+            const msg = /** @type {any} */ (event.data);
+            if (msg.type === 'progress') {
+                savedBytes = msg.reusedBytes;
+                if (onProgress && msg.totalBytes > 0) {
+                    const pct = Math.min(99, Math.round((100 * (msg.reusedBytes + msg.uploadedBytes)) / msg.totalBytes));
+                    onProgress(pct);
+                }
+                return;
+            }
+            if (msg.type === 'fallback') {
+                settle(null);
+                return;
+            }
+            if (msg.type === 'done') {
+                if (msg.status === 201 || msg.status === 200) {
+                    settle({ ok: true, data: msg.body, savedBytes });
+                    return;
+                }
+                /** @type {string} */
+                const errorMsg = msg.body?.message || msg.body?.error || `Delta upload failed (HTTP ${msg.status})`;
+                if (msg.status === 507) {
+                    settle({ ok: false, isQuotaError: true, errorMsg });
+                    return;
+                }
+                if (msg.status === 409 && !msg.body?.still_missing) {
+                    // Duplicate name — a byte upload would hit the same wall.
+                    settle({ ok: false, errorMsg });
+                    return;
+                }
+                // still_missing exhausted, 4xx/5xx oddities: byte upload is
+                // the safe road (the server dedups it on write anyway).
+                settle(null);
+            }
+        };
+        worker.onerror = () => {
+            // Worker script failed to load/parse — permanent environment trait.
+            _deltaUploadUsable = false;
+            settle(null);
+        };
+
+        worker.postMessage({
+            file,
+            folderId,
+            name: file.name,
+            csrfToken: getCsrfToken() || ''
+        });
+    });
+}
+
+/**
+ * Bilingual one-line summary for the bandwidth saved by a batch.
+ * @param {number} savedBytes
+ * @param {string} locale
+ * @returns {string}
+ */
+export function formatSavedSummary(savedBytes, locale) {
+    const mb = (savedBytes / (1024 * 1024)).toFixed(1);
+    return locale.startsWith('es') ? `Deduplicación: ${mb} MB no necesitaron subirse` : `Deduplication: ${mb} MB didn't need uploading`;
+}

static/js/features/files/fileOperations.js

@@ -12,7 +12,7 @@ import { i18n } from '../../core/i18n.js';
 import { notifications } from '../../core/notifications.js';
 import { invalidateFolderMeta } from '../../model/filesModel.js';
 import { triggerBrowserDownload } from '../../utils/download.js';
-import { tryInstantUpload } from './instantUpload.js';
+import { formatSavedSummary, tryDeltaUpload } from './deltaUpload.js';
 
 /**
  * @typedef {Object} BatchResult
@@ -392,6 +392,7 @@ const fileOps = {
             let uploadedCount = 0;
             let successCount = 0;
             let quotaStop = false;
+            let savedBytesTotal = 0;
 
             const targetFolderId = app.currentPath || app.userHomeFolderId;
 
@@ -405,12 +406,19 @@ const fileOps = {
                 if (quotaStop) return;
                 const file = readableFiles[idx];
 
-                // ── Instant upload: when the server already has this exact
-                // content for this user, register it by hash — zero bytes
-                // on the wire. Any miss/failure falls back to a byte upload.
-                /** @type {UploadAnswer | null} */
-                let result = await tryInstantUpload(file, targetFolderId);
+                // ── Delta upload: chunk + hash locally (worker/WASM) and
+                // transfer only what the server doesn't already have for
+                // this user. Any miss/failure falls back to a byte upload.
+                /** @type {UploadAnswer & { savedBytes?: number } | null} */
+                let result = await tryDeltaUpload(file, targetFolderId, (pct) => {
+                    if (batchId) {
+                        try {
+                            notifications.updateFile(batchId, file.name, pct, 'uploading');
+                        } catch (_) {}
+                    }
+                });
                 if (result) {
+                    savedBytesTotal += result.savedBytes || 0;
                     if (batchId) {
                         try {
                             notifications.updateFile(batchId, file.name, 100, result.ok ? 'done' : 'error');
@@ -492,6 +500,14 @@ const fileOps = {
 
             // All done
             this._finishUploadToast(successCount, totalFiles);
+            if (savedBytesTotal > 0 && notifications) {
+                notifications.addNotification({
+                    icon: 'fa-bolt',
+                    iconClass: 'upload',
+                    title: i18n?.getCurrentLocale?.()?.startsWith('es') ? 'Subida delta' : 'Delta upload',
+                    text: formatSavedSummary(savedBytesTotal, i18n?.getCurrentLocale?.() || 'en')
+                });
+            }
 
             // Refresh storage usage display
             try {
@@ -643,6 +659,7 @@ const fileOps = {
             let uploadedCount = 0;
             let successCount = 0;
             let quotaStop = false;
+            let savedBytesTotal = 0;
 
             // ── Concurrent upload with limited parallelism ──────────
             // FIFOs are pre-caught by the 0-byte arrayBuffer guard,
@@ -672,13 +689,14 @@ const fileOps = {
                     const parentPath = parts.slice(0, -1).join('/');
                     const targetFolderId = folderMap.get(parentPath) || currentFolderId;
 
-                    // ── Instant upload (zero bytes on the wire) ──
+                    // ── Delta upload (only changed bytes on the wire) ──
                     // Same fallback contract as uploadFiles: a null result
                     // means "do the byte upload". The shared accounting
                     // after this try block handles both outcomes.
-                    const instant = await tryInstantUpload(file, targetFolderId);
-                    if (instant) {
-                        result = instant;
+                    const delta = await tryDeltaUpload(file, targetFolderId);
+                    if (delta) {
+                        result = delta;
+                        savedBytesTotal += delta.savedBytes || 0;
                     } else {
                         // ── FIFO/pipe guard (0-byte files only) ──
                         // Named pipes (runit supervise/control) report size=0
@@ -773,6 +791,14 @@ const fileOps = {
             await Promise.all(workers);
 
             this._finishUploadToast(successCount, totalFiles);
+            if (savedBytesTotal > 0 && notifications) {
+                notifications.addNotification({
+                    icon: 'fa-bolt',
+                    iconClass: 'upload',
+                    title: i18n?.getCurrentLocale?.()?.startsWith('es') ? 'Subida delta' : 'Delta upload',
+                    text: formatSavedSummary(savedBytesTotal, i18n?.getCurrentLocale?.() || 'en')
+                });
+            }
 
             try {
                 await refreshUserData();

static/js/features/files/instantUpload.js

@@ -1,177 +0,0 @@
-/**
- * OxiCloud - Instant upload (zero-byte dedup upload)
- *
- * Before transferring a file's bytes, compute its BLAKE3 locally (in a
- * worker, off the main thread) and ask the server whether the caller
- * already owns that exact content (`GET /api/dedup/check/{hash}` — the
- * check is user-scoped, never a global content oracle). On a hit, a
- * single metadata call (`POST /api/files/by-hash`) registers the file
- * with ZERO content bytes on the wire.
- *
- * Performance posture:
- * - Hashing runs in a dedicated worker with WASM SIMD128 — the UI thread
- *   never blocks, RAM stays constant (8 MiB slices).
- * - Files below {@link INSTANT_UPLOAD_MIN_SIZE} skip the whole dance:
- *   two extra round-trips cost more than just uploading them.
- * - Any failure (no WASM support, worker error, server miss, races)
- *   falls back silently to the normal byte upload — instant upload is
- *   an optimization, never a gate.
- */
-
-import { getCsrfHeaders } from '../../core/csrf.js';
-
-/**
- * Files smaller than this upload normally: hashing + two round-trips
- * outweigh the transfer. 8 MiB matches the chunked-upload threshold's
- * order of magnitude.
- */
-export const INSTANT_UPLOAD_MIN_SIZE = 8 * 1024 * 1024;
-
-// Absolute URL on purpose — works in dev and in the release IIFE bundle
-// (same pattern as the pdf.js loader in thumbnail.js).
-const HASH_WORKER_URL = '/js/workers/hashWorker.js';
-
-/** Hashing budget: 60 s base + 30 s per GB (WASM SIMD does ~0.5-1 GB/s). */
-const HASH_TIMEOUT_BASE_MS = 60000;
-const HASH_TIMEOUT_PER_GB_MS = 30000;
-
-/**
- * `false` once the environment proved unable to run the worker/WASM
- * (old browser, blocked worker) — later files skip straight to the byte
- * upload instead of failing the same way again. `null` = not yet known.
- * @type {boolean | null}
- */
-let _instantUploadUsable = null;
-
-/**
- * Hash a file in a one-shot worker. Resolves `null` on any failure —
- * the caller falls back to a normal upload.
- * @param {File} file
- * @returns {Promise<string | null>}
- */
-function hashFileInWorker(file) {
-    return new Promise((resolve) => {
-        /** @type {Worker} */
-        let worker;
-        try {
-            worker = new Worker(HASH_WORKER_URL, { type: 'module' });
-        } catch (_) {
-            _instantUploadUsable = false;
-            resolve(null);
-            return;
-        }
-
-        const sizeGB = file.size / (1024 * 1024 * 1024);
-        const timeoutMs = HASH_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * HASH_TIMEOUT_PER_GB_MS;
-
-        /** @param {string | null} hash */
-        const settle = (hash) => {
-            clearTimeout(timer);
-            worker.terminate();
-            resolve(hash);
-        };
-        const timer = setTimeout(() => settle(null), timeoutMs);
-
-        worker.onmessage = (event) => {
-            const data = /** @type {{ ok: boolean, hash?: string, error?: string }} */ (event.data);
-            if (!data.ok) {
-                // The worker ran but WASM failed (e.g. no SIMD128 support):
-                // a permanent environment property, don't retry per file.
-                _instantUploadUsable = false;
-            }
-            settle(data.ok && data.hash ? data.hash : null);
-        };
-        worker.onerror = () => {
-            // Worker script failed to load/parse — permanent.
-            _instantUploadUsable = false;
-            settle(null);
-        };
-
-        worker.postMessage({ file });
-    });
-}
-
-/**
- * Ask the server whether the caller already owns content with this hash.
- * @param {string} hash
- * @returns {Promise<boolean>}
- */
-async function callerOwnsHash(hash) {
-    try {
-        const response = await fetch(`/api/dedup/check/${hash}`, {
-            headers: { 'Cache-Control': 'no-cache, no-store, must-revalidate' }
-        });
-        if (!response.ok) return false;
-        const body = /** @type {import('../../core/types.js').HashCheckAnswer} */ (await response.json());
-        return body.exists === true;
-    } catch (_) {
-        return false;
-    }
-}
-
-/**
- * Try to register `file` as a zero-byte instant upload.
- *
- * Returns `null` whenever the byte upload should proceed (file too
- * small, environment unusable, hash miss, lost race, transient errors).
- * Returns an upload-result object compatible with the uploaders'
- * `UploadAnswer` shape when the attempt is conclusive — success, quota
- * exceeded, or name conflict (a byte upload would fail identically).
- *
- * @param {File} file
- * @param {string | null | undefined} folderId
- * @returns {Promise<{ ok: boolean, data?: any, errorMsg?: string, isQuotaError?: boolean } | null>}
- */
-export async function tryInstantUpload(file, folderId) {
-    if (!folderId || file.size < INSTANT_UPLOAD_MIN_SIZE || _instantUploadUsable === false || typeof Worker === 'undefined') {
-        return null;
-    }
-
-    const hash = await hashFileInWorker(file);
-    if (!hash) return null;
-
-    if (!(await callerOwnsHash(hash))) return null;
-
-    try {
-        const response = await fetch('/api/files/by-hash', {
-            method: 'POST',
-            headers: {
-                ...getCsrfHeaders(),
-                'Content-Type': 'application/json',
-                'Cache-Control': 'no-cache, no-store, must-revalidate'
-            },
-            body: JSON.stringify(
-                /** @type {import('../../core/types.js').CreateFileByHash} */ ({
-                    name: file.name,
-                    folder_id: folderId,
-                    hash
-                })
-            )
-        });
-
-        if (response.status === 201) {
-            return { ok: true, data: await response.json() };
-        }
-
-        /** @type {string} */
-        let errorMsg = `Instant upload failed (HTTP ${response.status})`;
-        try {
-            const body = await response.json();
-            errorMsg = body.message || body.error || errorMsg;
-        } catch (_) {}
-
-        if (response.status === 507) {
-            return { ok: false, isQuotaError: true, errorMsg };
-        }
-        if (response.status === 409) {
-            // Duplicate name in the folder — a byte upload would hit the
-            // exact same conflict; surface it without transferring.
-            return { ok: false, errorMsg };
-        }
-        // 404 (ownership race with a delete+GC), 4xx/5xx: fall back to the
-        // byte upload — the server dedups it on write anyway.
-        return null;
-    } catch (_) {
-        return null;
-    }
-}