perf: serve ranges from RAM cache, stream ZIPs, overlap ingest settle, O(1) chunk gate

Round 2 of benchmark-gated optimizations (benches/ROUND2.md; every change
gated by a before/after in examples/bench_round2.rs — an AFTER that did
not beat its BEFORE was to be rolled back; none needed it):

- Range requests (REST/DAV/shares) answered from the moka content cache
  for sub-10MB files: PG resolve + open/seek/read -> Bytes::slice.
  256KiB seeks: 1,730/s -> 3.7M/s (p50 552us -> 0.15us).
- Streaming folder/share ZIPs via tokio duplex: TTFB no longer scales
  with archive size (326ms -> 0.4ms on 192MiB corpus; total also faster).
  Content-Length dropped (size unknown up front).
- NC chunked-upload per-PUT gate: O(k) directory scan+stat -> in-RAM
  per-session counter (lazy rebuild on cold start). 1,000-chunk upload
  gate cost: 33.1s -> 0.09s cumulative.
- Delta download + commit-verify now use the CDC path's
  buffered(read_prefetch) read-ahead: 64-chunk drain at 5ms open
  latency 440ms -> 51ms; order preserved.
- CDC ingest settles batches on a spawned task (depth-1 pipeline) so
  the source stream keeps flowing during PG pin + backend writes;
  rollback ledger shared + lock-serialized so compensation stays exact
  on cancellation. 512MiB paced ingest: 60-69 -> 74-75 MB/s.
  OXICLOUD_INGEST_OVERLAP=0 restores inline settling (ops/bench hatch).
- Frontend: instant-upload BLAKE3 hashing moved off the main thread to
  a bounded Web Worker pool (File handles by reference); vitest gate
  asserts the pool beats sequential (first gate draft posting buffers
  was 2.6x slower and was rewritten — copies dominated).

Validation: cargo fmt + clippy -D warnings clean; 514 unit + 544
integration tests green; 270 frontend tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w
This commit is contained in:
Claude
2026-07-16 16:50:07 +00:00
parent aba89c4f5d
commit 82ee7da0d2
18 changed files with 1245 additions and 195 deletions
@@ -0,0 +1,87 @@
import { describe, expect, it } from 'vitest';
import { Worker } from 'node:worker_threads';
import { createHash } from 'node:crypto';
import { promises as fs } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
/**
* Benchmark gate for the worker-pool hashing in `resolveOwnedHashes`.
*
* The browser change moves per-file BLAKE3 hashing from a sequential
* main-thread WASM loop onto a small pool of Web Workers. This test measures
* the same architecture on this machine with node's worker_threads and a
* CPU-bound digest as the stand-in workload: N buffers hashed sequentially
* on one thread vs the same work fanned over a 3-lane pool. If the pool
* doesn't beat sequential wall-clock, the frontend change must be rolled
* back (it would be pure complexity).
*/
describe('worker-pool hashing (architecture gate)', () => {
it('a 3-lane pool beats sequential main-thread hashing on wall clock', async () => {
// Faithful to the browser shape: the main thread hands each worker a
// FILE REFERENCE (browser: the File handle; here: its path) and the
// worker does read + hash. The old shape reads + hashes every file
// on the main thread, serially.
const nFiles = 24;
const size = 4 * 1024 * 1024;
const dir = await fs.mkdtemp(join(tmpdir(), 'hashbench-'));
const paths: string[] = [];
for (let i = 0; i < nFiles; i++) {
const p = join(dir, `f${i}`);
const b = Buffer.alloc(size);
b.fill(i + 1);
await fs.writeFile(p, b);
paths.push(p);
}
// Sequential (old): read + hash on the calling thread.
const t0 = performance.now();
for (const p of paths) {
const b = await fs.readFile(p);
createHash('sha256').update(b).digest('hex');
}
const seqMs = performance.now() - t0;
// 3-lane pool (new): each worker reads + hashes its own files.
const lanes = 3;
const workerSrc = `
const { parentPort } = require('node:worker_threads');
const { createHash } = require('node:crypto');
const { readFileSync } = require('node:fs');
parentPort.on('message', (path) => {
const b = readFileSync(path);
parentPort.postMessage(createHash('sha256').update(b).digest('hex'));
});
`;
const workers = Array.from({ length: lanes }, () => new Worker(workerSrc, { eval: true }));
let next = 0;
const t1 = performance.now();
await Promise.all(
workers.map(
(w) =>
new Promise<void>((resolve, reject) => {
const feed = () => {
if (next >= paths.length) {
resolve();
return;
}
const i = next++;
w.once('message', () => feed());
w.once('error', reject);
w.postMessage(paths[i]);
};
feed();
})
)
);
const poolMs = performance.now() - t1;
await Promise.all(workers.map((w) => w.terminate()));
await fs.rm(dir, { recursive: true, force: true });
// eslint-disable-next-line no-console
console.info(
`read+hash ${nFiles} x 4 MiB: sequential ${seqMs.toFixed(0)} ms vs 3-lane pool ${poolMs.toFixed(0)} ms (${(seqMs / poolMs).toFixed(1)}x)`
);
expect(poolMs).toBeLessThan(seqMs);
});
});
+60 -1
View File
@@ -176,6 +176,59 @@ export async function instantUploadOwned(
return null;
}
const HASH_WORKER_URL = '/workers/hashWorker.js';
/** Parallel hashing lanes — enough to saturate small-file hashing without
* starving the upload workers of cores. */
const HASH_POOL_SIZE = Math.min(4, Math.max(1, (navigator.hardwareConcurrency ?? 2) - 1));
/**
* BLAKE3-hash `files` on a bounded pool of dedicated workers (main thread
* stays free). A file whose worker errors is simply absent from the result —
* the caller uploads it the normal way. Falls back to the sequential inline
* hasher when `Worker` is unavailable.
*/
async function hashFilesPooled(files: File[]): Promise<Map<File, string>> {
if (typeof Worker === 'undefined') {
const out = new Map<File, string>();
for (const f of files) out.set(f, await blake3HexOfFile(f));
return out;
}
const lanes = Math.min(HASH_POOL_SIZE, files.length);
const workers = Array.from(
{ length: lanes },
() => new Worker(HASH_WORKER_URL, { type: 'module' })
);
const out = new Map<File, string>();
let next = 0;
try {
await Promise.all(
workers.map(
(w) =>
new Promise<void>((resolve, reject) => {
const feed = () => {
if (next >= files.length) {
resolve();
return;
}
const i = next++;
const file = files[i];
w.onmessage = (ev: MessageEvent<{ id: number; hex?: string; error?: string }>) => {
if (ev.data.hex) out.set(file, ev.data.hex);
feed(); // per-file errors: skip the file, keep the lane
};
w.onerror = (e) => reject(e);
w.postMessage({ id: i, file });
};
feed();
})
)
);
} finally {
for (const w of workers) w.terminate();
}
return out;
}
/**
* Resolve which of `files` the server already owns, with a SINGLE batch round
* trip (the Dropbox-style "have you got these?" probe). Every file below the
@@ -193,7 +246,13 @@ export async function resolveOwnedHashes(files: File[]): Promise<Map<File, strin
const hashByFile = new Map<File, string>();
try {
for (const f of inBand) hashByFile.set(f, await blake3HexOfFile(f));
// Hash off the main thread on a small worker pool — the sequential
// main-thread WASM loop blocked the UI for the whole batch and
// delayed every upload lane behind the full hashing phase (measured
// in deltaUpload.hash.test.ts). Falls back to the inline loop when
// Workers are unavailable (some test environments).
const hashed = await hashFilesPooled(inBand);
for (const [f, h] of hashed) hashByFile.set(f, h);
} catch {
return new Map(); // WASM/hashing unavailable → skip instant uploads
}