visual continunity

This commit is contained in:
Bradley Nelson
2026-06-17 22:07:18 -06:00
parent daa3010458
commit 89e14f8f9e
89 changed files with 19249 additions and 1367 deletions
+343
View File
@@ -0,0 +1,343 @@
/**
* Incremental BLAKE3 hasher.
*
* ```js
* const h = new Blake3Hasher();
* h.update(chunkBytes); // repeat per slice
* const hex = h.finalizeHex();
* ```
*/
export class Blake3Hasher {
__destroy_into_raw() {
const ptr = this.__wbg_ptr;
this.__wbg_ptr = 0;
Blake3HasherFinalization.unregister(this);
return ptr;
}
free() {
const ptr = this.__destroy_into_raw();
wasm.__wbg_blake3hasher_free(ptr, 0);
}
/**
* Bytes hashed so far — lets the worker report progress without
* tracking its own counter.
* @returns {number}
*/
count() {
const ret = wasm.blake3hasher_count(this.__wbg_ptr);
return ret;
}
/**
* Finish and return the lowercase hex digest (64 chars). The hasher
* can keep receiving `update` calls afterwards (BLAKE3 finalization
* is non-destructive), but the frontend treats it as terminal.
* @returns {string}
*/
finalizeHex() {
let deferred1_0;
let deferred1_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.blake3hasher_finalizeHex(retptr, this.__wbg_ptr);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
deferred1_0 = r0;
deferred1_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_export2(deferred1_0, deferred1_1, 1);
}
}
/**
* Create a fresh hasher.
*/
constructor() {
const ret = wasm.blake3hasher_new();
this.__wbg_ptr = ret;
Blake3HasherFinalization.register(this, this.__wbg_ptr, this);
return this;
}
/**
* Feed one slice of the file.
* @param {Uint8Array} data
*/
update(data) {
const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export);
const len0 = WASM_VECTOR_LEN;
wasm.blake3hasher_update(this.__wbg_ptr, ptr0, len0);
}
}
if (Symbol.dispose) Blake3Hasher.prototype[Symbol.dispose] = Blake3Hasher.prototype.free;
/**
* Incremental FastCDC chunker + whole-file BLAKE3, for the delta-upload
* worker. Feed the file in slices; every call returns the chunks that
* became FINAL; `finish()` flushes the tail and returns the file hash.
*
* ```js
* const c = new DeltaChunker();
* for (const slice of slices) {
* for (const [h, s] of JSON.parse(c.update(bytes))) { … }
* }
* const { chunks, file_hash } = JSON.parse(c.finish());
* ```
*
* Correctness of the incremental split: FastCDC decides each cut by
* scanning at most `CDC_MAX_CHUNK` bytes from the chunk's start. When
* the chunker runs over the buffered prefix of a longer file, every
* produced chunk except the LAST ended on a content/max-size condition
* — its decision window was fully available, so the full-file chunker
* makes the same cut. Only the last chunk (cut by "end of buffer") is
* provisional: it stays buffered and is re-examined when more bytes
* arrive. By induction the emitted boundaries equal a single FastCDC
* pass over the whole file — the mirror test below proves it.
*/
export class DeltaChunker {
__destroy_into_raw() {
const ptr = this.__wbg_ptr;
this.__wbg_ptr = 0;
DeltaChunkerFinalization.unregister(this);
return ptr;
}
free() {
const ptr = this.__destroy_into_raw();
wasm.__wbg_deltachunker_free(ptr, 0);
}
/**
* Flush the provisional tail and return
* `{"chunks":[["<hex>",size]…],"file_hash":"<hex>","total":N}`.
* `chunks` holds at most one entry (the tail); an empty file has none
* and its `file_hash` is BLAKE3 of the empty input.
* @returns {string}
*/
finish() {
let deferred1_0;
let deferred1_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
wasm.deltachunker_finish(retptr, this.__wbg_ptr);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
deferred1_0 = r0;
deferred1_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_export2(deferred1_0, deferred1_1, 1);
}
}
/**
* Create a chunker with the server's CDC parameters.
*/
constructor() {
const ret = wasm.deltachunker_new();
this.__wbg_ptr = ret;
DeltaChunkerFinalization.register(this, this.__wbg_ptr, this);
return this;
}
/**
* Feed one slice. Returns a JSON array of the chunks that became
* final: `[["<blake3-hex>", size], …]` (possibly empty).
* @param {Uint8Array} data
* @returns {string}
*/
update(data) {
let deferred2_0;
let deferred2_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export);
const len0 = WASM_VECTOR_LEN;
wasm.deltachunker_update(retptr, this.__wbg_ptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_export2(deferred2_0, deferred2_1, 1);
}
}
}
if (Symbol.dispose) DeltaChunker.prototype[Symbol.dispose] = DeltaChunker.prototype.free;
/**
* One-shot convenience for small buffers.
* @param {Uint8Array} data
* @returns {string}
*/
export function blake3Hex(data) {
let deferred2_0;
let deferred2_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export);
const len0 = WASM_VECTOR_LEN;
wasm.blake3Hex(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_export2(deferred2_0, deferred2_1, 1);
}
}
function __wbg_get_imports() {
const import0 = {
__proto__: null,
__wbg___wbindgen_throw_bbadd78c1bac3a77: function(arg0, arg1) {
throw new Error(getStringFromWasm0(arg0, arg1));
},
};
return {
__proto__: null,
"./oxicloud_hash_wasm_bg.js": import0,
};
}
const Blake3HasherFinalization = (typeof FinalizationRegistry === 'undefined')
? { register: () => {}, unregister: () => {} }
: new FinalizationRegistry(ptr => wasm.__wbg_blake3hasher_free(ptr, 1));
const DeltaChunkerFinalization = (typeof FinalizationRegistry === 'undefined')
? { register: () => {}, unregister: () => {} }
: new FinalizationRegistry(ptr => wasm.__wbg_deltachunker_free(ptr, 1));
let cachedDataViewMemory0 = null;
function getDataViewMemory0() {
if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
}
return cachedDataViewMemory0;
}
function getStringFromWasm0(ptr, len) {
return decodeText(ptr >>> 0, len);
}
let cachedUint8ArrayMemory0 = null;
function getUint8ArrayMemory0() {
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8ArrayMemory0;
}
function passArray8ToWasm0(arg, malloc) {
const ptr = malloc(arg.length * 1, 1) >>> 0;
getUint8ArrayMemory0().set(arg, ptr / 1);
WASM_VECTOR_LEN = arg.length;
return ptr;
}
let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
cachedTextDecoder.decode();
const MAX_SAFARI_DECODE_BYTES = 2146435072;
let numBytesDecoded = 0;
function decodeText(ptr, len) {
numBytesDecoded += len;
if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
cachedTextDecoder.decode();
numBytesDecoded = len;
}
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
}
let WASM_VECTOR_LEN = 0;
let wasmModule, wasmInstance, wasm;
function __wbg_finalize_init(instance, module) {
wasmInstance = instance;
wasm = instance.exports;
wasmModule = module;
cachedDataViewMemory0 = null;
cachedUint8ArrayMemory0 = null;
return wasm;
}
async function __wbg_load(module, imports) {
if (typeof Response === 'function' && module instanceof Response) {
if (typeof WebAssembly.instantiateStreaming === 'function') {
try {
return await WebAssembly.instantiateStreaming(module, imports);
} catch (e) {
const validResponse = module.ok && expectedResponseType(module.type);
if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') {
console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
} else { throw e; }
}
}
const bytes = await module.arrayBuffer();
return await WebAssembly.instantiate(bytes, imports);
} else {
const instance = await WebAssembly.instantiate(module, imports);
if (instance instanceof WebAssembly.Instance) {
return { instance, module };
} else {
return instance;
}
}
function expectedResponseType(type) {
switch (type) {
case 'basic': case 'cors': case 'default': return true;
}
return false;
}
}
function initSync(module) {
if (wasm !== undefined) return wasm;
if (module !== undefined) {
if (Object.getPrototypeOf(module) === Object.prototype) {
({module} = module)
} else {
console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
}
}
const imports = __wbg_get_imports();
if (!(module instanceof WebAssembly.Module)) {
module = new WebAssembly.Module(module);
}
const instance = new WebAssembly.Instance(module, imports);
return __wbg_finalize_init(instance, module);
}
async function __wbg_init(module_or_path) {
if (wasm !== undefined) return wasm;
if (module_or_path !== undefined) {
if (Object.getPrototypeOf(module_or_path) === Object.prototype) {
({module_or_path} = module_or_path)
} else {
console.warn('using deprecated parameters for the initialization function; pass a single object instead')
}
}
if (module_or_path === undefined) {
module_or_path = new URL('oxicloud_hash_wasm_bg.wasm', import.meta.url);
}
const imports = __wbg_get_imports();
if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
module_or_path = fetch(module_or_path);
}
const { instance, module } = await __wbg_load(await module_or_path, imports);
return __wbg_finalize_init(instance, module);
}
export { initSync, __wbg_init as default };
Binary file not shown.
+330
View File
@@ -0,0 +1,330 @@
/**
* OxiCloud — delta-upload worker ("upload only what changed").
*
* Runs the whole client side of the delta protocol off the main thread:
*
* read 8 MiB slices ─► FastCDC chunk + BLAKE3 (WASM, same crate and
* parameters as the server) ─► negotiate hash batches ─► upload only
* the missing chunks (framed, bounded concurrency) ─► commit.
*
* The stages OVERLAP: negotiation of batch N and uploads of its missing
* chunks run while batch N+1 is still being hashed, so wall-clock time
* approaches max(hash time, upload time) instead of their sum. RAM stays
* flat: chunk bytes are re-sliced from the File at upload time, never
* hoarded.
*
* Protocol with the spawner:
* in : { file: File, folderId: string, name: string, csrfToken: string }
* out : { type: 'progress', hashedBytes, reusedBytes, uploadedBytes, totalBytes }
* { type: 'done', status, body } — conclusive HTTP outcome
* { type: 'fallback', reason } — do a plain byte upload
*/
// Absolute URLs on purpose: vendors/workers are served verbatim in both
// dev and the static build (served verbatim from /static).
const WASM_GLUE_URL = '/vendors/hash-wasm/oxicloud_hash_wasm.js';
/** File read granularity — large enough to amortize Blob→ArrayBuffer. */
const SLICE_BYTES = 8 * 1024 * 1024;
/** Negotiate after this many freshly hashed chunks (~64 MiB of content). */
const NEGOTIATE_BATCH = 256;
/** Group missing chunks into PUT bodies of at most this many bytes. */
const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024;
/** Concurrent chunk-PUT requests. */
const UPLOAD_CONCURRENCY = 2;
/** Re-commit attempts when the server answers 409 still_missing. */
const COMMIT_RETRIES = 2;
/**
* Typed view of the dedicated-worker global scope (jsconfig targets the
* DOM lib, where `self` is a Window — cast to what this worker uses).
* @type {{ onmessage: ((event: MessageEvent) => void) | null,
* postMessage: (message: unknown) => void }}
*/
const workerScope = /** @type {any} */ (self);
/**
* One chunk occurrence, in file order.
* @typedef {{ h: string, s: number, offset: number }} WorkerChunk
*/
/** @returns {Promise<any>} the initialized WASM module */
async function loadWasm() {
const mod = await import(WASM_GLUE_URL);
await mod.default();
return mod;
}
workerScope.onmessage = async (event) => {
const { file, folderId, name, csrfToken } = /** @type {{ file: File, folderId: string, name: string, csrfToken: string }} */ (event.data);
/** @param {string} reason */
const fallback = (reason) => workerScope.postMessage({ type: 'fallback', reason });
/** @type {Record<string, string>} */
const mutHeaders = { 'Content-Type': 'application/json' };
if (csrfToken) mutHeaders['X-CSRF-Token'] = csrfToken;
let wasm;
try {
wasm = await loadWasm();
} catch (err) {
fallback(`wasm unavailable: ${err instanceof Error ? err.message : String(err)}`);
return;
}
// ── Shared pipeline state ─────────────────────────────────────
/** @type {WorkerChunk[]} */
const chunks = []; // every occurrence, in file order
/** @type {Set<string>} */
const seenForNegotiate = new Set(); // distinct hashes already sent to negotiate
let reusedBytes = 0;
let uploadedBytes = 0;
let hashedBytes = 0;
let failed = /** @type {string | null} */ (null);
let lastProgress = 0;
const progress = (force = false) => {
const now = Date.now();
if (!force && now - lastProgress < 150) return;
lastProgress = now;
workerScope.postMessage({
type: 'progress',
hashedBytes,
reusedBytes,
uploadedBytes,
totalBytes: file.size
});
};
// ── Upload stage: bounded-concurrency drain of uploadByHash ──
/** @type {WorkerChunk[]} */
const uploadQueue = [];
/** @type {Promise<void>[]} */
const uploadWorkers = [];
let uploadsClosed = false;
/** @type {(() => void) | null} */
let wakeUploader = null;
const signalUploaders = () => {
if (wakeUploader) {
const w = wakeUploader;
wakeUploader = null;
w();
}
};
/** Encode a batch of chunks as [u32 BE len][bytes] frames. */
const encodeFrames = async (/** @type {WorkerChunk[]} */ batch) => {
const total = batch.reduce((n, c) => n + 4 + c.s, 0);
const wire = new Uint8Array(total);
const view = new DataView(wire.buffer);
let at = 0;
for (const c of batch) {
// eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM
const bytes = new Uint8Array(await file.slice(c.offset, c.offset + c.s).arrayBuffer());
view.setUint32(at, c.s, false);
wire.set(bytes, at + 4);
at += 4 + c.s;
}
return wire;
};
const uploadLoop = async () => {
while (!failed) {
// Take up to UPLOAD_BATCH_BYTES from the queue.
/** @type {WorkerChunk[]} */
const batch = [];
let bytes = 0;
while (uploadQueue.length > 0 && bytes < UPLOAD_BATCH_BYTES) {
const c = /** @type {WorkerChunk} */ (uploadQueue.shift());
batch.push(c);
bytes += c.s;
}
if (batch.length === 0) {
if (uploadsClosed) return;
// eslint-disable-next-line no-await-in-loop -- queue wait
await new Promise((resolve) => {
wakeUploader = /** @type {() => void} */ (resolve);
});
continue;
}
try {
// eslint-disable-next-line no-await-in-loop -- bounded by pool size
const wire = await encodeFrames(batch);
// eslint-disable-next-line no-await-in-loop -- bounded by pool size
const response = await fetch('/api/files/delta/chunks', {
method: 'PUT',
headers: {
'Content-Type': 'application/octet-stream',
...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {})
},
body: wire
});
if (!response.ok) {
failed = `chunk PUT failed (HTTP ${response.status})`;
return;
}
for (const c of batch) uploadedBytes += c.s;
progress();
} catch (err) {
failed = `chunk PUT failed: ${err instanceof Error ? err.message : String(err)}`;
return;
}
}
};
for (let i = 0; i < UPLOAD_CONCURRENCY; i++) uploadWorkers.push(uploadLoop());
// ── Negotiate stage ───────────────────────────────────────────
/** @type {Promise<void>[]} */
const negotiations = [];
const negotiate = (/** @type {WorkerChunk[]} */ fresh) => {
if (fresh.length === 0 || failed) return;
negotiations.push(
(async () => {
try {
const response = await fetch('/api/files/delta/negotiate', {
method: 'POST',
headers: mutHeaders,
body: JSON.stringify({ chunks: fresh.map(({ h, s }) => ({ h, s })) })
});
if (!response.ok) {
failed = failed || `negotiate failed (HTTP ${response.status})`;
return;
}
const missing = new Set(/** @type {{missing: string[]}} */ (await response.json()).missing);
for (const c of fresh) {
if (missing.has(c.h)) {
uploadQueue.push(c);
} else {
reusedBytes += c.s;
}
}
signalUploaders();
progress();
} catch (err) {
failed = failed || `negotiate failed: ${err instanceof Error ? err.message : String(err)}`;
}
})()
);
};
// ── Chunking stage (drives the other two) ────────────────────
try {
const chunker = new wasm.DeltaChunker();
/** @type {WorkerChunk[]} */
let freshBatch = [];
let offset = 0;
/** @param {[string, number][]} emitted */
const onChunks = (emitted) => {
for (const [h, s] of emitted) {
/** @type {WorkerChunk} */
const chunk = { h, s, offset };
offset += s;
chunks.push(chunk);
if (seenForNegotiate.has(h)) {
// Repeated content inside the same file: the first
// occurrence decides upload vs reuse; later ones are
// pure reuse for accounting.
reusedBytes += s;
} else {
seenForNegotiate.add(h);
freshBatch.push(chunk);
if (freshBatch.length >= NEGOTIATE_BATCH) {
negotiate(freshBatch);
freshBatch = [];
}
}
}
};
for (let read = 0; read < file.size && !failed; read += SLICE_BYTES) {
const end = Math.min(read + SLICE_BYTES, file.size);
// eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM
const slice = new Uint8Array(await file.slice(read, end).arrayBuffer());
onChunks(JSON.parse(chunker.update(slice)));
hashedBytes = end;
progress();
}
const fin = JSON.parse(chunker.finish());
chunker.free();
onChunks(fin.chunks);
negotiate(freshBatch);
const fileHash = /** @type {string} */ (fin.file_hash);
hashedBytes = file.size;
progress(true);
// ── Drain: negotiations → uploads → commit ───────────────
await Promise.all(negotiations);
uploadsClosed = true;
signalUploaders();
await Promise.all(uploadWorkers);
if (failed) {
fallback(failed);
return;
}
const commitBody = {
file_hash: fileHash,
chunks: chunks.map(({ h, s }) => ({ h, s })),
name,
folder_id: folderId
};
for (let attempt = 0; ; attempt++) {
// eslint-disable-next-line no-await-in-loop -- retry loop
const response = await fetch('/api/files/delta/commit', {
method: 'POST',
headers: mutHeaders,
body: JSON.stringify(commitBody)
});
/** @type {any} */
let body = null;
try {
// eslint-disable-next-line no-await-in-loop -- retry loop
body = await response.json();
} catch (_) {}
const stillMissing = response.status === 409 && Array.isArray(body?.still_missing);
if (stillMissing && attempt < COMMIT_RETRIES) {
// GC race or a chunk we wrongly assumed claimable: upload
// exactly what the server names and try again.
const byHash = new Map(chunks.map((c) => [c.h, c]));
/** @type {WorkerChunk[]} */
const retry = [];
for (const h of body.still_missing) {
const c = byHash.get(h);
if (!c) {
fallback('server requested an unknown chunk');
return;
}
retry.push(c);
}
const wire = await encodeFrames(retry);
// eslint-disable-next-line no-await-in-loop -- retry loop
const put = await fetch('/api/files/delta/chunks', {
method: 'PUT',
headers: {
'Content-Type': 'application/octet-stream',
...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {})
},
body: wire
});
if (!put.ok) {
fallback(`retry chunk PUT failed (HTTP ${put.status})`);
return;
}
for (const c of retry) uploadedBytes += c.s;
progress(true);
continue;
}
// Conclusive: 201 created, or a real error (quota, name
// conflict, validation). The spawner maps it to the uploaders'
// UploadAnswer contract.
workerScope.postMessage({ type: 'done', status: response.status, body });
return;
}
} catch (err) {
fallback(err instanceof Error ? err.message : String(err));
}
};