Instant upload: register already-owned content by hash, zero bytes on the wire
Phase 0 of the delta-sync plan. Re-uploading a file the user already has (another device, a restore, a duplicate) used to transfer every byte just for the server to discard them as a dedup hit. The frontend now computes the file's BLAKE3 locally and, on a hit, registers the file with a single ~150-byte metadata call. Server — POST /api/files/by-hash: - All checks live in the application service per the AuthZ rule: Create permission on the target folder via the authorization engine, hash ownership via the existing user-scoped query (a non-owned hash returns 404 — same shape as "no such blob" — and emits an instant_upload.rejected audit event), quota on the logical size. - On success: one ref_count bump + the existing save_file_with_blob row registration (compensation included); is_new_blob=false so lifecycle hooks skip thumbnail regeneration. ~10 ms warm. - The storage-usage service is now built before the application services and injected, instead of only living on AppState. Client — WASM BLAKE3 + worker: - wasm/oxicloud-hash: the exact same blake3 crate the server uses, compiled with WASM SIMD128 (~660 MB/s measured) so browser hashes match server content addresses bit for bit. Built by scripts/build-wasm.sh; the artifacts (45 KB wasm + 8 KB glue) are vendored like pdf.js — no npm dependencies, no wasm toolchain needed for regular builds. - static/js/workers/hashWorker.js streams the File in 8 MiB slices off the main thread (constant RAM at any file size). - features/files/instantUpload.js orchestrates: threshold (8 MiB — below it the round-trips cost more than the bytes), user-scoped /api/dedup/check, by-hash registration, and silent fallback to the normal byte upload on any miss, race or unsupported environment. Wired into both uploadFiles and uploadFolderEntries. - biome.json vendors exclusion fixed to cover nested directories (previous vendors were .mjs and never matched the *.js include). Verified end-to-end against PostgreSQL 16: node-driven WASM hash equals the server's content_hash for a 20 MB file; by-hash returns 201 in ~10 ms warm with a 151-byte request (vs 20,971,873 bytes for the byte upload); the copy downloads byte-identical and the manifest ref_count goes 1→2; a second user probing the same hash gets exists:false and 404 plus the audit line; duplicate name → 409, malformed hash → 400; worker and wasm are served with correct MIME (application/wasm). https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS
This commit is contained in:
@@ -518,3 +518,26 @@
|
||||
* @typedef {{kind: 'user', id: string} | {kind: 'group', id: string}} GroupMemberItem
|
||||
*/
|
||||
|
||||
// ------------------- Instant upload (dedup)
|
||||
|
||||
/**
|
||||
* Response from `GET /api/dedup/check/{hash}` — user-scoped: `exists` only
|
||||
* reflects content the CALLER already owns, never global existence.
|
||||
* Mirrors `HashCheckResponse` on the server (`dedup_handler.rs`).
|
||||
* @typedef {Object} HashCheckAnswer
|
||||
* @property {boolean} exists
|
||||
* @property {string} hash BLAKE3 echoed back (64 hex chars)
|
||||
* @property {number} [existing_size] size in bytes, present when `exists`
|
||||
*/
|
||||
|
||||
/**
|
||||
* Request body for `POST /api/files/by-hash` (instant upload — registers a
|
||||
* file from an already-owned blob, zero content bytes on the wire).
|
||||
* Mirrors `CreateFileByHashRequest` on the server (`file_handler.rs`).
|
||||
* The 201 response body is a {@link FileItem}.
|
||||
* @typedef {Object} CreateFileByHash
|
||||
* @property {string} name file name to create (basename only)
|
||||
* @property {string} folder_id target folder (caller needs Create on it)
|
||||
* @property {string} hash BLAKE3 of the owned content (64 hex chars)
|
||||
*/
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import { i18n } from '../../core/i18n.js';
|
||||
import { notifications } from '../../core/notifications.js';
|
||||
import { invalidateFolderMeta } from '../../model/filesModel.js';
|
||||
import { triggerBrowserDownload } from '../../utils/download.js';
|
||||
import { tryInstantUpload } from './instantUpload.js';
|
||||
|
||||
/**
|
||||
* @typedef {Object} BatchResult
|
||||
@@ -404,20 +405,28 @@ const fileOps = {
|
||||
if (quotaStop) return;
|
||||
const file = readableFiles[idx];
|
||||
|
||||
const formData = new FormData();
|
||||
if (targetFolderId) formData.append('folder_id', targetFolderId);
|
||||
formData.append('file', file);
|
||||
// ── Instant upload: when the server already has this exact
|
||||
// content for this user, register it by hash — zero bytes
|
||||
// on the wire. Any miss/failure falls back to a byte upload.
|
||||
/** @type {UploadAnswer | null} */
|
||||
let result = await tryInstantUpload(file, targetFolderId);
|
||||
if (result) {
|
||||
if (batchId) {
|
||||
try {
|
||||
notifications.updateFile(batchId, file.name, 100, result.ok ? 'done' : 'error');
|
||||
} catch (_) {}
|
||||
}
|
||||
} else {
|
||||
const formData = new FormData();
|
||||
if (targetFolderId) formData.append('folder_id', targetFolderId);
|
||||
formData.append('file', file);
|
||||
|
||||
console.log(`Uploading file to folder: ${targetFolderId || 'root'}`, {
|
||||
file: file.name,
|
||||
size: file.size
|
||||
});
|
||||
|
||||
// Scale stall timeout with file size:
|
||||
// base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit
|
||||
const sizeGB = file.size / (1024 * 1024 * 1024);
|
||||
const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000);
|
||||
const result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout);
|
||||
// Scale stall timeout with file size:
|
||||
// base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit
|
||||
const sizeGB = file.size / (1024 * 1024 * 1024);
|
||||
const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000);
|
||||
result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout);
|
||||
}
|
||||
|
||||
uploadedCount++;
|
||||
|
||||
@@ -663,48 +672,54 @@ const fileOps = {
|
||||
const parentPath = parts.slice(0, -1).join('/');
|
||||
const targetFolderId = folderMap.get(parentPath) || currentFolderId;
|
||||
|
||||
// ── FIFO/pipe guard (0-byte files only) ──
|
||||
// Named pipes (runit supervise/control) report size=0
|
||||
// but block on open(). Pre-read only 0-byte files into
|
||||
// memory; files with size>0 are always regular files and
|
||||
// go straight to FormData (zero extra memory copy).
|
||||
/** @type {Blob} */
|
||||
let uploadFile = file; // default: use original File
|
||||
if (file.size === 0) {
|
||||
try {
|
||||
const buf = await Promise.race([
|
||||
file.arrayBuffer(),
|
||||
new Promise((_, rej) => setTimeout(() => rej(new Error('read-timeout')), 2000))
|
||||
]);
|
||||
uploadFile = new Blob([buf], {
|
||||
type: file.type || 'application/octet-stream'
|
||||
});
|
||||
} catch {
|
||||
console.warn(`[SKIP] #${idx} ${rel} — cannot read 0-byte file (FIFO/pipe?), skipping`);
|
||||
uploadedCount++;
|
||||
successCount++;
|
||||
if (batchId) {
|
||||
try {
|
||||
notifications.fileCompleted(batchId, true);
|
||||
} catch (_) {}
|
||||
// ── Instant upload (zero bytes on the wire) ──
|
||||
// Same fallback contract as uploadFiles: a null result
|
||||
// means "do the byte upload". The shared accounting
|
||||
// after this try block handles both outcomes.
|
||||
const instant = await tryInstantUpload(file, targetFolderId);
|
||||
if (instant) {
|
||||
result = instant;
|
||||
} else {
|
||||
// ── FIFO/pipe guard (0-byte files only) ──
|
||||
// Named pipes (runit supervise/control) report size=0
|
||||
// but block on open(). Pre-read only 0-byte files into
|
||||
// memory; files with size>0 are always regular files and
|
||||
// go straight to FormData (zero extra memory copy).
|
||||
/** @type {Blob} */
|
||||
let uploadFile = file; // default: use original File
|
||||
if (file.size === 0) {
|
||||
try {
|
||||
const buf = await Promise.race([
|
||||
file.arrayBuffer(),
|
||||
new Promise((_, rej) => setTimeout(() => rej(new Error('read-timeout')), 2000))
|
||||
]);
|
||||
uploadFile = new Blob([buf], {
|
||||
type: file.type || 'application/octet-stream'
|
||||
});
|
||||
} catch {
|
||||
console.warn(`[SKIP] #${idx} ${rel} — cannot read 0-byte file (FIFO/pipe?), skipping`);
|
||||
uploadedCount++;
|
||||
successCount++;
|
||||
if (batchId) {
|
||||
try {
|
||||
notifications.fileCompleted(batchId, true);
|
||||
} catch (_) {}
|
||||
}
|
||||
return;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('folder_id', targetFolderId);
|
||||
formData.append('file', uploadFile, file.name);
|
||||
|
||||
const thisTimeout =
|
||||
file.size === 0
|
||||
? TIMEOUT_MS_ZERO
|
||||
: Math.max(TIMEOUT_MIN_MS, TIMEOUT_BASE_MS + Math.ceil(file.size / (1024 * 1024)) * TIMEOUT_PER_MB_MS);
|
||||
|
||||
result = await this._uploadFileFetch(formData, thisTimeout);
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('folder_id', targetFolderId);
|
||||
formData.append('file', uploadFile, file.name);
|
||||
|
||||
const thisTimeout =
|
||||
file.size === 0
|
||||
? TIMEOUT_MS_ZERO
|
||||
: Math.max(TIMEOUT_MIN_MS, TIMEOUT_BASE_MS + Math.ceil(file.size / (1024 * 1024)) * TIMEOUT_PER_MB_MS);
|
||||
console.log(`[UPLOAD START] #${idx} ${rel} (${file.size} bytes, timeout=${thisTimeout}ms)`);
|
||||
|
||||
result = await this._uploadFileFetch(formData, thisTimeout);
|
||||
|
||||
console.log(`[UPLOAD END] #${idx} ${rel} ok=${result.ok}${result.errorMsg ? ` err=${result.errorMsg}` : ''}`);
|
||||
} catch (e) {
|
||||
result = {
|
||||
ok: false,
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
/**
|
||||
* OxiCloud - Instant upload (zero-byte dedup upload)
|
||||
*
|
||||
* Before transferring a file's bytes, compute its BLAKE3 locally (in a
|
||||
* worker, off the main thread) and ask the server whether the caller
|
||||
* already owns that exact content (`GET /api/dedup/check/{hash}` — the
|
||||
* check is user-scoped, never a global content oracle). On a hit, a
|
||||
* single metadata call (`POST /api/files/by-hash`) registers the file
|
||||
* with ZERO content bytes on the wire.
|
||||
*
|
||||
* Performance posture:
|
||||
* - Hashing runs in a dedicated worker with WASM SIMD128 — the UI thread
|
||||
* never blocks, RAM stays constant (8 MiB slices).
|
||||
* - Files below {@link INSTANT_UPLOAD_MIN_SIZE} skip the whole dance:
|
||||
* two extra round-trips cost more than just uploading them.
|
||||
* - Any failure (no WASM support, worker error, server miss, races)
|
||||
* falls back silently to the normal byte upload — instant upload is
|
||||
* an optimization, never a gate.
|
||||
*/
|
||||
|
||||
import { getCsrfHeaders } from '../../core/csrf.js';
|
||||
|
||||
/**
|
||||
* Files smaller than this upload normally: hashing + two round-trips
|
||||
* outweigh the transfer. 8 MiB matches the chunked-upload threshold's
|
||||
* order of magnitude.
|
||||
*/
|
||||
export const INSTANT_UPLOAD_MIN_SIZE = 8 * 1024 * 1024;
|
||||
|
||||
// Absolute URL on purpose — works in dev and in the release IIFE bundle
|
||||
// (same pattern as the pdf.js loader in thumbnail.js).
|
||||
const HASH_WORKER_URL = '/js/workers/hashWorker.js';
|
||||
|
||||
/** Hashing budget: 60 s base + 30 s per GB (WASM SIMD does ~0.5-1 GB/s). */
|
||||
const HASH_TIMEOUT_BASE_MS = 60000;
|
||||
const HASH_TIMEOUT_PER_GB_MS = 30000;
|
||||
|
||||
/**
|
||||
* `false` once the environment proved unable to run the worker/WASM
|
||||
* (old browser, blocked worker) — later files skip straight to the byte
|
||||
* upload instead of failing the same way again. `null` = not yet known.
|
||||
* @type {boolean | null}
|
||||
*/
|
||||
let _instantUploadUsable = null;
|
||||
|
||||
/**
|
||||
* Hash a file in a one-shot worker. Resolves `null` on any failure —
|
||||
* the caller falls back to a normal upload.
|
||||
* @param {File} file
|
||||
* @returns {Promise<string | null>}
|
||||
*/
|
||||
function hashFileInWorker(file) {
|
||||
return new Promise((resolve) => {
|
||||
/** @type {Worker} */
|
||||
let worker;
|
||||
try {
|
||||
worker = new Worker(HASH_WORKER_URL, { type: 'module' });
|
||||
} catch (_) {
|
||||
_instantUploadUsable = false;
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
const sizeGB = file.size / (1024 * 1024 * 1024);
|
||||
const timeoutMs = HASH_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * HASH_TIMEOUT_PER_GB_MS;
|
||||
|
||||
/** @param {string | null} hash */
|
||||
const settle = (hash) => {
|
||||
clearTimeout(timer);
|
||||
worker.terminate();
|
||||
resolve(hash);
|
||||
};
|
||||
const timer = setTimeout(() => settle(null), timeoutMs);
|
||||
|
||||
worker.onmessage = (event) => {
|
||||
const data = /** @type {{ ok: boolean, hash?: string, error?: string }} */ (event.data);
|
||||
if (!data.ok) {
|
||||
// The worker ran but WASM failed (e.g. no SIMD128 support):
|
||||
// a permanent environment property, don't retry per file.
|
||||
_instantUploadUsable = false;
|
||||
}
|
||||
settle(data.ok && data.hash ? data.hash : null);
|
||||
};
|
||||
worker.onerror = () => {
|
||||
// Worker script failed to load/parse — permanent.
|
||||
_instantUploadUsable = false;
|
||||
settle(null);
|
||||
};
|
||||
|
||||
worker.postMessage({ file });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the server whether the caller already owns content with this hash.
|
||||
* @param {string} hash
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
async function callerOwnsHash(hash) {
|
||||
try {
|
||||
const response = await fetch(`/api/dedup/check/${hash}`, {
|
||||
headers: { 'Cache-Control': 'no-cache, no-store, must-revalidate' }
|
||||
});
|
||||
if (!response.ok) return false;
|
||||
const body = /** @type {import('../../core/types.js').HashCheckAnswer} */ (await response.json());
|
||||
return body.exists === true;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to register `file` as a zero-byte instant upload.
|
||||
*
|
||||
* Returns `null` whenever the byte upload should proceed (file too
|
||||
* small, environment unusable, hash miss, lost race, transient errors).
|
||||
* Returns an upload-result object compatible with the uploaders'
|
||||
* `UploadAnswer` shape when the attempt is conclusive — success, quota
|
||||
* exceeded, or name conflict (a byte upload would fail identically).
|
||||
*
|
||||
* @param {File} file
|
||||
* @param {string | null | undefined} folderId
|
||||
* @returns {Promise<{ ok: boolean, data?: any, errorMsg?: string, isQuotaError?: boolean } | null>}
|
||||
*/
|
||||
export async function tryInstantUpload(file, folderId) {
|
||||
if (!folderId || file.size < INSTANT_UPLOAD_MIN_SIZE || _instantUploadUsable === false || typeof Worker === 'undefined') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const hash = await hashFileInWorker(file);
|
||||
if (!hash) return null;
|
||||
|
||||
if (!(await callerOwnsHash(hash))) return null;
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/files/by-hash', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
...getCsrfHeaders(),
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-cache, no-store, must-revalidate'
|
||||
},
|
||||
body: JSON.stringify(
|
||||
/** @type {import('../../core/types.js').CreateFileByHash} */ ({
|
||||
name: file.name,
|
||||
folder_id: folderId,
|
||||
hash
|
||||
})
|
||||
)
|
||||
});
|
||||
|
||||
if (response.status === 201) {
|
||||
return { ok: true, data: await response.json() };
|
||||
}
|
||||
|
||||
/** @type {string} */
|
||||
let errorMsg = `Instant upload failed (HTTP ${response.status})`;
|
||||
try {
|
||||
const body = await response.json();
|
||||
errorMsg = body.message || body.error || errorMsg;
|
||||
} catch (_) {}
|
||||
|
||||
if (response.status === 507) {
|
||||
return { ok: false, isQuotaError: true, errorMsg };
|
||||
}
|
||||
if (response.status === 409) {
|
||||
// Duplicate name in the folder — a byte upload would hit the
|
||||
// exact same conflict; surface it without transferring.
|
||||
return { ok: false, errorMsg };
|
||||
}
|
||||
// 404 (ownership race with a delete+GC), 4xx/5xx: fall back to the
|
||||
// byte upload — the server dedups it on write anyway.
|
||||
return null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Incremental BLAKE3 hasher.
|
||||
*
|
||||
* ```js
|
||||
* const h = new Blake3Hasher();
|
||||
* h.update(chunkBytes); // repeat per slice
|
||||
* const hex = h.finalizeHex();
|
||||
* ```
|
||||
*/
|
||||
export class Blake3Hasher {
|
||||
__destroy_into_raw() {
|
||||
const ptr = this.__wbg_ptr;
|
||||
this.__wbg_ptr = 0;
|
||||
Blake3HasherFinalization.unregister(this);
|
||||
return ptr;
|
||||
}
|
||||
free() {
|
||||
const ptr = this.__destroy_into_raw();
|
||||
wasm.__wbg_blake3hasher_free(ptr, 0);
|
||||
}
|
||||
/**
|
||||
* Bytes hashed so far — lets the worker report progress without
|
||||
* tracking its own counter.
|
||||
* @returns {number}
|
||||
*/
|
||||
count() {
|
||||
const ret = wasm.blake3hasher_count(this.__wbg_ptr);
|
||||
return ret;
|
||||
}
|
||||
/**
|
||||
* Finish and return the lowercase hex digest (64 chars). The hasher
|
||||
* can keep receiving `update` calls afterwards (BLAKE3 finalization
|
||||
* is non-destructive), but the frontend treats it as terminal.
|
||||
* @returns {string}
|
||||
*/
|
||||
finalizeHex() {
|
||||
let deferred1_0;
|
||||
let deferred1_1;
|
||||
try {
|
||||
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
||||
wasm.blake3hasher_finalizeHex(retptr, this.__wbg_ptr);
|
||||
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
||||
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
||||
deferred1_0 = r0;
|
||||
deferred1_1 = r1;
|
||||
return getStringFromWasm0(r0, r1);
|
||||
} finally {
|
||||
wasm.__wbindgen_add_to_stack_pointer(16);
|
||||
wasm.__wbindgen_export2(deferred1_0, deferred1_1, 1);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Create a fresh hasher.
|
||||
*/
|
||||
constructor() {
|
||||
const ret = wasm.blake3hasher_new();
|
||||
this.__wbg_ptr = ret;
|
||||
Blake3HasherFinalization.register(this, this.__wbg_ptr, this);
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Feed one slice of the file.
|
||||
* @param {Uint8Array} data
|
||||
*/
|
||||
update(data) {
|
||||
const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
wasm.blake3hasher_update(this.__wbg_ptr, ptr0, len0);
|
||||
}
|
||||
}
|
||||
if (Symbol.dispose) Blake3Hasher.prototype[Symbol.dispose] = Blake3Hasher.prototype.free;
|
||||
|
||||
/**
|
||||
* One-shot convenience for small buffers.
|
||||
* @param {Uint8Array} data
|
||||
* @returns {string}
|
||||
*/
|
||||
export function blake3Hex(data) {
|
||||
let deferred2_0;
|
||||
let deferred2_1;
|
||||
try {
|
||||
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
|
||||
const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export);
|
||||
const len0 = WASM_VECTOR_LEN;
|
||||
wasm.blake3Hex(retptr, ptr0, len0);
|
||||
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
|
||||
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
|
||||
deferred2_0 = r0;
|
||||
deferred2_1 = r1;
|
||||
return getStringFromWasm0(r0, r1);
|
||||
} finally {
|
||||
wasm.__wbindgen_add_to_stack_pointer(16);
|
||||
wasm.__wbindgen_export2(deferred2_0, deferred2_1, 1);
|
||||
}
|
||||
}
|
||||
function __wbg_get_imports() {
|
||||
const import0 = {
|
||||
__proto__: null,
|
||||
__wbg___wbindgen_throw_bbadd78c1bac3a77: (arg0, arg1) => {
|
||||
throw new Error(getStringFromWasm0(arg0, arg1));
|
||||
}
|
||||
};
|
||||
return {
|
||||
__proto__: null,
|
||||
'./oxicloud_hash_wasm_bg.js': import0
|
||||
};
|
||||
}
|
||||
|
||||
const Blake3HasherFinalization =
|
||||
typeof FinalizationRegistry === 'undefined'
|
||||
? { register: () => {}, unregister: () => {} }
|
||||
: new FinalizationRegistry((ptr) => wasm.__wbg_blake3hasher_free(ptr, 1));
|
||||
|
||||
let cachedDataViewMemory0 = null;
|
||||
function getDataViewMemory0() {
|
||||
if (
|
||||
cachedDataViewMemory0 === null ||
|
||||
cachedDataViewMemory0.buffer.detached === true ||
|
||||
(cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)
|
||||
) {
|
||||
cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
|
||||
}
|
||||
return cachedDataViewMemory0;
|
||||
}
|
||||
|
||||
function getStringFromWasm0(ptr, len) {
|
||||
return decodeText(ptr >>> 0, len);
|
||||
}
|
||||
|
||||
let cachedUint8ArrayMemory0 = null;
|
||||
function getUint8ArrayMemory0() {
|
||||
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
|
||||
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
|
||||
}
|
||||
return cachedUint8ArrayMemory0;
|
||||
}
|
||||
|
||||
function passArray8ToWasm0(arg, malloc) {
|
||||
const ptr = malloc(arg.length * 1, 1) >>> 0;
|
||||
getUint8ArrayMemory0().set(arg, ptr / 1);
|
||||
WASM_VECTOR_LEN = arg.length;
|
||||
return ptr;
|
||||
}
|
||||
|
||||
let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
||||
cachedTextDecoder.decode();
|
||||
const MAX_SAFARI_DECODE_BYTES = 2146435072;
|
||||
let numBytesDecoded = 0;
|
||||
function decodeText(ptr, len) {
|
||||
numBytesDecoded += len;
|
||||
if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
|
||||
cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
|
||||
cachedTextDecoder.decode();
|
||||
numBytesDecoded = len;
|
||||
}
|
||||
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
|
||||
}
|
||||
|
||||
let WASM_VECTOR_LEN = 0;
|
||||
|
||||
let wasmModule, wasmInstance, wasm;
|
||||
function __wbg_finalize_init(instance, module) {
|
||||
wasmInstance = instance;
|
||||
wasm = instance.exports;
|
||||
wasmModule = module;
|
||||
cachedDataViewMemory0 = null;
|
||||
cachedUint8ArrayMemory0 = null;
|
||||
return wasm;
|
||||
}
|
||||
|
||||
async function __wbg_load(module, imports) {
|
||||
if (typeof Response === 'function' && module instanceof Response) {
|
||||
if (typeof WebAssembly.instantiateStreaming === 'function') {
|
||||
try {
|
||||
return await WebAssembly.instantiateStreaming(module, imports);
|
||||
} catch (e) {
|
||||
const validResponse = module.ok && expectedResponseType(module.type);
|
||||
|
||||
if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') {
|
||||
console.warn(
|
||||
'`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n',
|
||||
e
|
||||
);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const bytes = await module.arrayBuffer();
|
||||
return await WebAssembly.instantiate(bytes, imports);
|
||||
} else {
|
||||
const instance = await WebAssembly.instantiate(module, imports);
|
||||
|
||||
if (instance instanceof WebAssembly.Instance) {
|
||||
return { instance, module };
|
||||
} else {
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
|
||||
function expectedResponseType(type) {
|
||||
switch (type) {
|
||||
case 'basic':
|
||||
case 'cors':
|
||||
case 'default':
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function initSync(module) {
|
||||
if (wasm !== undefined) return wasm;
|
||||
|
||||
if (module !== undefined) {
|
||||
if (Object.getPrototypeOf(module) === Object.prototype) {
|
||||
({ module } = module);
|
||||
} else {
|
||||
console.warn('using deprecated parameters for `initSync()`; pass a single object instead');
|
||||
}
|
||||
}
|
||||
|
||||
const imports = __wbg_get_imports();
|
||||
if (!(module instanceof WebAssembly.Module)) {
|
||||
module = new WebAssembly.Module(module);
|
||||
}
|
||||
const instance = new WebAssembly.Instance(module, imports);
|
||||
return __wbg_finalize_init(instance, module);
|
||||
}
|
||||
|
||||
async function __wbg_init(module_or_path) {
|
||||
if (wasm !== undefined) return wasm;
|
||||
|
||||
if (module_or_path !== undefined) {
|
||||
if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
|
||||
({ module_or_path } = module_or_path);
|
||||
} else {
|
||||
console.warn('using deprecated parameters for the initialization function; pass a single object instead');
|
||||
}
|
||||
}
|
||||
|
||||
if (module_or_path === undefined) {
|
||||
module_or_path = new URL('oxicloud_hash_wasm_bg.wasm', import.meta.url);
|
||||
}
|
||||
const imports = __wbg_get_imports();
|
||||
|
||||
if (
|
||||
typeof module_or_path === 'string' ||
|
||||
(typeof Request === 'function' && module_or_path instanceof Request) ||
|
||||
(typeof URL === 'function' && module_or_path instanceof URL)
|
||||
) {
|
||||
module_or_path = fetch(module_or_path);
|
||||
}
|
||||
|
||||
const { instance, module } = await __wbg_load(await module_or_path, imports);
|
||||
|
||||
return __wbg_finalize_init(instance, module);
|
||||
}
|
||||
|
||||
export { __wbg_init as default, initSync };
|
||||
Binary file not shown.
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* OxiCloud — BLAKE3 hashing worker (instant-upload support).
|
||||
*
|
||||
* Hashes a File off the main thread, reading it in fixed-size slices so
|
||||
* RAM stays constant regardless of file size. The WASM module is compiled
|
||||
* from the exact same `blake3` crate the server uses, so the digest
|
||||
* computed here equals the server's content address bit for bit.
|
||||
*
|
||||
* Protocol: receives `{ file: File }`, answers
|
||||
* `{ ok: true, hash: string }` or `{ ok: false, error: string }`.
|
||||
* The spawner terminates the worker after one file.
|
||||
*/
|
||||
|
||||
// Absolute URL on purpose: vendors are served verbatim at /js/vendors/ in
|
||||
// both dev and release mode (the release IIFE bundle would break a
|
||||
// relative import) — same pattern as the pdf.js loader in thumbnail.js.
|
||||
const WASM_GLUE_URL = '/js/vendors/hash-wasm/oxicloud_hash_wasm.js';
|
||||
|
||||
/**
|
||||
* 8 MiB slices — large enough to amortize the per-slice Blob→ArrayBuffer
|
||||
* round-trip, small enough that peak worker RAM stays flat for any size.
|
||||
*/
|
||||
const SLICE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Typed view of the dedicated-worker global scope. The project's
|
||||
* jsconfig targets the DOM lib, where `self` is a Window — cast to the
|
||||
* two members this worker actually uses.
|
||||
* @type {{ onmessage: ((event: MessageEvent) => void) | null,
|
||||
* postMessage: (message: unknown) => void }}
|
||||
*/
|
||||
const workerScope = /** @type {any} */ (self);
|
||||
|
||||
/**
|
||||
* Memoized WASM module (in-flight or settled), `default()` already run.
|
||||
* Reset on failure so a later message can retry a transient load error.
|
||||
* @type {Promise<any> | null}
|
||||
*/
|
||||
let _wasmPromise = null;
|
||||
|
||||
/** @returns {Promise<any>} */
|
||||
function getWasm() {
|
||||
if (!_wasmPromise) {
|
||||
_wasmPromise = import(WASM_GLUE_URL)
|
||||
.then(async (mod) => {
|
||||
await mod.default();
|
||||
return mod;
|
||||
})
|
||||
.catch((err) => {
|
||||
_wasmPromise = null;
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
return _wasmPromise;
|
||||
}
|
||||
|
||||
workerScope.onmessage = async (event) => {
|
||||
const file = /** @type {{ file: File }} */ (event.data).file;
|
||||
try {
|
||||
const wasm = await getWasm();
|
||||
const hasher = new wasm.Blake3Hasher();
|
||||
try {
|
||||
for (let offset = 0; offset < file.size; offset += SLICE_BYTES) {
|
||||
const end = Math.min(offset + SLICE_BYTES, file.size);
|
||||
// eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM
|
||||
const buffer = await file.slice(offset, end).arrayBuffer();
|
||||
hasher.update(new Uint8Array(buffer));
|
||||
}
|
||||
workerScope.postMessage({ ok: true, hash: hasher.finalizeHex() });
|
||||
} finally {
|
||||
hasher.free();
|
||||
}
|
||||
} catch (err) {
|
||||
workerScope.postMessage({
|
||||
ok: false,
|
||||
error: err instanceof Error ? err.message : String(err)
|
||||
});
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user