Merge pull request #674 from EdouardVanbelle/feat/ui-log-namespaced
Feat/UI log namespaced
This commit is contained in:
Dionisio Pozo
@@ -175,6 +175,7 @@ export default defineConfig({
|
||||
{ text: "Authentication model", link: "/architecture/auth-model" },
|
||||
{ text: "Magic-link auth", link: "/architecture/magic-link-auth" },
|
||||
{ text: "Background jobs", link: "/architecture/jobs" },
|
||||
{ text: "UI diagnostics", link: "/architecture/ui-diagnostics" },
|
||||
],
|
||||
},
|
||||
{ text: "FAQ", link: "/faq" },
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
# UI Diagnostics
|
||||
|
||||
Runtime-tunable knobs and namespaced logging exposed to the browser
|
||||
DevTools console so upload and session issues can be diagnosed without
|
||||
a rebuild, a config flag, or a page reload.
|
||||
|
||||
All entry points live under a single global — `window.oxi` — attached
|
||||
during the client init hook (`frontend/src/hooks.client.ts`). Type
|
||||
`oxi.` in DevTools with autocomplete to discover what's available; the
|
||||
sections below name what each surface does.
|
||||
|
||||
## The `oxi.*` helper
|
||||
|
||||
```js
|
||||
oxi.log // raw `loglevel` module
|
||||
oxi.setLogLevel(ns, level) // toggle a namespace's level
|
||||
oxi.listLogLevels() // enumerate persisted overrides
|
||||
oxi.UPLOAD_BATCH_BYTES // get/set per-PUT byte cap for delta upload
|
||||
```
|
||||
|
||||
`oxi.setLogLevel` returns a confirmation string (`"oxi:upload → debug"`)
|
||||
so the DevTools echo is a positive signal instead of the confusing
|
||||
`undefined` a naked `void`-returning setter would produce.
|
||||
|
||||
## Log levels
|
||||
|
||||
The frontend uses [`loglevel`](https://github.com/pimterry/loglevel)
|
||||
with per-namespace levels. Each subsystem gets its own logger; toggle
|
||||
them independently.
|
||||
|
||||
### Namespaces
|
||||
|
||||
| Namespace | Emitted by |
|
||||
| ----------- | ------------------------------------------------------------------ |
|
||||
| `oxi:upload` | Delta + direct upload pipeline (`lib/api/endpoints/deltaUpload.ts`, `static/workers/deltaWorker.js`) |
|
||||
|
||||
New namespaces should follow the `oxi:<subsystem>` shape so a wildcard
|
||||
filter across the whole app remains meaningful.
|
||||
|
||||
### Levels
|
||||
|
||||
`trace` < `debug` < `info` < `warn` < `error` < `silent`.
|
||||
|
||||
Default per namespace is `info` — phase transitions and error paths
|
||||
surface without any opt-in. Bump to `debug` for per-chunk /
|
||||
per-batch verbose trace during a failure hunt.
|
||||
|
||||
### Runtime toggle
|
||||
|
||||
```js
|
||||
// Deep dive into upload internals
|
||||
oxi.setLogLevel('oxi:upload', 'debug')
|
||||
// → 'oxi:upload → debug'
|
||||
|
||||
// Quiet mode — only warnings and errors
|
||||
oxi.setLogLevel('oxi:upload', 'warn')
|
||||
|
||||
// Full silence
|
||||
oxi.setLogLevel('oxi:upload', 'silent')
|
||||
|
||||
// See every namespace's current override
|
||||
oxi.listLogLevels()
|
||||
// → { 'oxi:upload': 'DEBUG' }
|
||||
|
||||
// Everything (including future namespaces) to debug
|
||||
oxi.log.setLevel('debug')
|
||||
```
|
||||
|
||||
Changes persist to `localStorage` under the key
|
||||
`loglevel:<namespace>` — the choice survives page reloads and browser
|
||||
restarts until you explicitly change it back or clear localStorage.
|
||||
|
||||
Worker context: `deltaWorker.js` runs in a Web Worker and can't
|
||||
`import 'loglevel'` (the worker is served from `/static` without
|
||||
bundler resolution). Instead it emits log events via `postMessage` and
|
||||
the main-thread orchestrator relays them through the shared logger, so
|
||||
`oxi.setLogLevel('oxi:upload', 'debug')` filters worker output too.
|
||||
|
||||
## Upload batch tuning
|
||||
|
||||
`oxi.UPLOAD_BATCH_BYTES` controls the target size of each `PUT
|
||||
/api/files/delta/chunks` body — the delta worker groups missing chunks
|
||||
into that size before sending. Default is 8 MiB; can go up (fewer,
|
||||
larger requests) or down (more, smaller requests).
|
||||
|
||||
```js
|
||||
oxi.UPLOAD_BATCH_BYTES // read current value
|
||||
// → 8388608
|
||||
|
||||
oxi.UPLOAD_BATCH_BYTES = 1024 * 1024 // 1 MiB per PUT
|
||||
// → 1048576
|
||||
|
||||
oxi.UPLOAD_BATCH_BYTES = 8 * 1024 * 1024 // back to default (removes the override)
|
||||
```
|
||||
|
||||
Persisted to `localStorage['oxi:upload:batchBytes']`. Setting the value
|
||||
back to the default clears the entry so the storage stays clean.
|
||||
|
||||
### When to lower it
|
||||
|
||||
Behind reverse proxies with tight per-request timeouts. The classic
|
||||
case is **Cloudflare Tunnel**: 100-second absolute per-request
|
||||
timeout on the Free/Pro plans. A user on a slow uplink (say, hotel
|
||||
Wi-Fi at 512 Kbps) can't complete an 8 MiB PUT in that window and gets
|
||||
their request cut mid-flight. Lower to 1 MiB (~16 seconds at 512 Kbps)
|
||||
and it fits comfortably.
|
||||
|
||||
Cost: about 8× more HTTP requests per file. TCP keep-alive amortises
|
||||
most of the connection setup; the extra CPU is negligible.
|
||||
|
||||
### Read timing
|
||||
|
||||
Read once per upload at worker spawn time. Change from the console →
|
||||
the NEXT upload picks up the new value; the currently-running upload
|
||||
finishes with the old value. No reload required.
|
||||
|
||||
## Delta upload trace
|
||||
|
||||
A healthy fresh upload at `info` level looks like:
|
||||
|
||||
```
|
||||
[3f7a2b] delta start {file: "vacation.mp4", size: 524288000}
|
||||
[3f7a2b] worker: worker start {file: "vacation.mp4", size: 524288000}
|
||||
[3f7a2b] worker: wasm loaded
|
||||
[3f7a2b] worker: hashed — blake3=<64-hex> (512 chunks)
|
||||
[3f7a2b] worker: negotiate: 256 hashes → 240 missing, 16 dedup'd
|
||||
[3f7a2b] worker: negotiate: 256 hashes → 256 missing, 0 dedup'd
|
||||
[3f7a2b] worker: ✅ committed — uploaded 501346304 B, reused 22941696 B (4% dedup, blake3=<hash>)
|
||||
[3f7a2b] worker: commit HTTP 201 {blake3, uploadedBytes, reusedBytes, totalBytes, attempt}
|
||||
[3f7a2b] delta done {file, blake3, savedBytes, uploadedBytes}
|
||||
```
|
||||
|
||||
At `debug` level the worker additionally emits one line per chunk PUT
|
||||
(`chunk PUT: 28 chunks, 8825338 bytes`) — expect roughly one line per
|
||||
`UPLOAD_BATCH_BYTES` worth of body sent.
|
||||
|
||||
Every line prefixes a short 6-hex upload id (`3f7a2b`) so concurrent
|
||||
uploads stay distinguishable in the console. The `blake3` field is the
|
||||
whole-file BLAKE3 hash the commit call carried — same value as
|
||||
`storage.file_blobs.hash` on the server, so log lines correlate
|
||||
directly to server-side blob rows.
|
||||
|
||||
### Common failure signatures
|
||||
|
||||
| Log line | Meaning | What to try |
|
||||
| --------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `worker fallback: wasm unavailable: <detail>` | Browser blocked WebAssembly (CSP, extension, disabled in prefs) | Check CSP `script-src 'wasm-unsafe-eval'` allows it; disable content-blocking extensions on the tab |
|
||||
| `delta disabled for this tab: Worker constructor threw` | `new Worker(...)` failed — usually CSP `worker-src` | Check CSP `worker-src` allows `blob:` if the worker uses one, or the origin |
|
||||
| `worker: chunk PUT failed (HTTP 413)` → `worker requested fallback` | Server (or a proxy in front of it) rejected the PUT body as too large | Lower `oxi.UPLOAD_BATCH_BYTES`; check proxy body caps |
|
||||
| `worker: chunk PUT failed: <NetworkError / timeout>` | Cloudflare (or another proxy) cut the request mid-flight | Lower `oxi.UPLOAD_BATCH_BYTES` so each PUT fits inside the proxy's per-request timeout |
|
||||
| `worker: negotiate failed (HTTP 5xx)` | Server-side error during the negotiate stage | Server logs; grep for the emitted `request_id` |
|
||||
| `delta worker went silent for 20s — disabling delta for this tab` | Worker stopped emitting progress — WASM hang, DoS, or extreme main-thread contention. Poisons this tab. | Reload to reset the poison flag; check `console` for errors emitted by the WASM module or the worker itself |
|
||||
| `delta timeout after Xs — falling back to direct upload` | Wall-clock timeout (`120s + 90s per GB`) — usually means chunk PUTs are stalling | Look for the last `chunk PUT` line; if none appeared for many seconds, network is stalled at the tunnel |
|
||||
| `delta done with non-2xx (HTTP 500) — falling back to direct upload` | Commit rejected server-side — chunk verification, quota, name conflict, etc. | Server logs; look for `delta_upload.rejected` audit line with a `reason` field |
|
||||
| `worker: commit HTTP 507` | Storage quota exceeded | User needs to free space or admin needs to increase quota |
|
||||
|
||||
The generic pattern for user reports: ask them to open DevTools →
|
||||
Console tab (filter: `oxi:upload`), run `oxi.setLogLevel('oxi:upload',
|
||||
'debug')`, retry the failing upload, and share the output. The last
|
||||
line before the "Upload failed" toast names the actual failure.
|
||||
|
||||
## Interrupted uploads
|
||||
|
||||
Two coordinated behaviours help users recover from an accidental page
|
||||
reload during an upload (details in
|
||||
`frontend/src/lib/upload/interruption.ts`):
|
||||
|
||||
### `beforeunload` guard
|
||||
|
||||
While any upload is in flight, the browser prompts *"Leave site?
|
||||
Changes may not be saved"* on refresh / tab close. Deliberate leave
|
||||
(user clicks Leave) proceeds; accidental Cmd-R is cancelled.
|
||||
|
||||
Installed lazily: the listener is added when the first batch acquires
|
||||
the guard, removed when the last batch releases it. No cost outside
|
||||
active uploads.
|
||||
|
||||
### sessionStorage register
|
||||
|
||||
Every `uploadBatch` writes a record to
|
||||
`sessionStorage['oxi:upload:interrupted']` while it runs and removes
|
||||
it on completion. If a reload survives them, the root layout's
|
||||
`onMount` reads and clears the register, then toasts:
|
||||
|
||||
> Upload interrupted: `<name>`. Re-drop to resume — already-uploaded
|
||||
> chunks are reused.
|
||||
|
||||
Chunks landed in `storage.file_blobs` before the reload persist on the
|
||||
server. A re-drop lets the delta worker's `negotiate` stage discover
|
||||
them as "already on server", so a resumed upload only transfers what
|
||||
was in flight when the reload hit — not everything from scratch.
|
||||
|
||||
`sessionStorage` (not `localStorage`) on purpose: entries clear when
|
||||
the tab closes entirely, so a user who closed the tab hours ago
|
||||
doesn't get nagged on reopen.
|
||||
|
||||
## Code entry points
|
||||
|
||||
| Concern | File |
|
||||
| ----------------------------------- | --------------------------------------------------------------- |
|
||||
| `oxi.*` global attach + setters | `frontend/src/hooks.client.ts` |
|
||||
| Delta orchestrator (main thread) | `frontend/src/lib/api/endpoints/deltaUpload.ts` |
|
||||
| Delta worker (CDC + BLAKE3 + PUTs) | `frontend/static/workers/deltaWorker.js` |
|
||||
| Direct upload (fallback path) | `frontend/src/lib/api/endpoints/files.ts::uploadFileWithProgress` |
|
||||
| Interrupted-upload registry | `frontend/src/lib/upload/interruption.ts` |
|
||||
| Reload-time toast wiring | `frontend/src/routes/+layout.svelte` (`onMount`) |
|
||||
|
||||
Server-side counterpart for delta upload:
|
||||
[Delta upload protocol](../delta-upload-protocol.md).
|
||||
Generated
+15
-1
@@ -8,7 +8,8 @@
|
||||
"name": "oxicloud-frontend",
|
||||
"version": "0.0.0",
|
||||
"dependencies": {
|
||||
"@serenity-kit/opaque": "^1.1.0"
|
||||
"@serenity-kit/opaque": "^1.1.0",
|
||||
"loglevel": "^1.9.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^10.0.1",
|
||||
@@ -3969,6 +3970,19 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/loglevel": {
|
||||
"version": "1.9.2",
|
||||
"resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz",
|
||||
"integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "tidelift",
|
||||
"url": "https://tidelift.com/funding/github/npm/loglevel"
|
||||
}
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "11.5.1",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz",
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
"vitest": "^4.1.9"
|
||||
},
|
||||
"dependencies": {
|
||||
"@serenity-kit/opaque": "^1.1.0"
|
||||
"@serenity-kit/opaque": "^1.1.0",
|
||||
"loglevel": "^1.9.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,12 +3,106 @@
|
||||
* - wires the API client's session-expired behaviour (clear store + redirect),
|
||||
* - loads translations for the resolved locale.
|
||||
*/
|
||||
import log from 'loglevel';
|
||||
import { setSessionExpiredHandler } from '$lib/api/client';
|
||||
import { initI18n } from '$lib/i18n/index.svelte';
|
||||
import { session } from '$lib/stores/session.svelte';
|
||||
import { seedNonceFromCookie } from '$lib/auth/dpop-proof';
|
||||
|
||||
// DevTools shortcut: expose a small `oxi.*` helper on window so users
|
||||
// can toggle log levels and knobs from the browser console without
|
||||
// needing to import anything.
|
||||
//
|
||||
// Log levels — namespaces used today: `oxi:upload` (delta + direct
|
||||
// upload pipeline). Levels: 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'silent'.
|
||||
// Choices persist to `localStorage['loglevel:<namespace>']` via loglevel.
|
||||
//
|
||||
// oxi.setLogLevel('oxi:upload', 'debug') // deep dive
|
||||
// oxi.setLogLevel('oxi:upload', 'warn') // quiet
|
||||
// oxi.log.setLevel('debug') // everything to debug
|
||||
//
|
||||
// Delta-upload batch size — bytes per PUT to `/api/files/delta/chunks`.
|
||||
// Default 8 MiB. Behind proxies with tight per-request timeouts
|
||||
// (Cloudflare Tunnel: 100 s absolute), lower this so each PUT completes
|
||||
// within the window on a slow uplink:
|
||||
//
|
||||
// oxi.UPLOAD_BATCH_BYTES = 1024 * 1024 // 1 MiB per PUT
|
||||
//
|
||||
// Persists to `localStorage['oxi:upload:batchBytes']`. Read on every
|
||||
// upload — set once from the console, refresh not required.
|
||||
const BATCH_BYTES_KEY = 'oxi:upload:batchBytes';
|
||||
const BATCH_BYTES_DEFAULT = 8 * 1024 * 1024;
|
||||
|
||||
function readBatchBytes(): number {
|
||||
try {
|
||||
if (typeof localStorage === 'undefined') return BATCH_BYTES_DEFAULT;
|
||||
const raw = localStorage.getItem(BATCH_BYTES_KEY);
|
||||
if (!raw) return BATCH_BYTES_DEFAULT;
|
||||
const n = Number(raw);
|
||||
return Number.isFinite(n) && n > 0 ? n : BATCH_BYTES_DEFAULT;
|
||||
} catch {
|
||||
return BATCH_BYTES_DEFAULT;
|
||||
}
|
||||
}
|
||||
|
||||
function writeBatchBytes(n: number): void {
|
||||
if (typeof localStorage === 'undefined') return;
|
||||
try {
|
||||
if (n === BATCH_BYTES_DEFAULT) localStorage.removeItem(BATCH_BYTES_KEY);
|
||||
else localStorage.setItem(BATCH_BYTES_KEY, String(n));
|
||||
} catch {
|
||||
/* quota / disabled — best-effort */
|
||||
}
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
oxi?: {
|
||||
log: typeof log;
|
||||
setLogLevel: (namespace: string, level: log.LogLevelDesc) => string;
|
||||
listLogLevels: () => Record<string, string>;
|
||||
UPLOAD_BATCH_BYTES: number;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function init(): Promise<void> {
|
||||
if (typeof window !== 'undefined') {
|
||||
const helpers = {
|
||||
log,
|
||||
// Return a confirmation string so the DevTools echo is a
|
||||
// useful "worked → new level" signal instead of `undefined`.
|
||||
setLogLevel(namespace: string, level: log.LogLevelDesc): string {
|
||||
log.getLogger(namespace).setLevel(level);
|
||||
return `${namespace} → ${level}`;
|
||||
},
|
||||
// Enumerate the levels loglevel has persisted so users can see
|
||||
// what's currently set without opening the Application tab.
|
||||
listLogLevels(): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
if (typeof localStorage === 'undefined') return out;
|
||||
for (let i = 0; i < localStorage.length; i++) {
|
||||
const key = localStorage.key(i);
|
||||
if (key?.startsWith('loglevel:')) {
|
||||
out[key.slice('loglevel:'.length)] = localStorage.getItem(key) ?? '';
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
};
|
||||
// UPLOAD_BATCH_BYTES: getter reads live from localStorage so any
|
||||
// tab / component pulling `window.oxi.UPLOAD_BATCH_BYTES` sees the
|
||||
// current value; setter persists so the choice survives reload
|
||||
// (mirrors loglevel's persistence pattern).
|
||||
Object.defineProperty(helpers, 'UPLOAD_BATCH_BYTES', {
|
||||
get: readBatchBytes,
|
||||
set: writeBatchBytes,
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
});
|
||||
window.oxi = helpers as Window['oxi'];
|
||||
}
|
||||
|
||||
setSessionExpiredHandler(() => {
|
||||
session.reset();
|
||||
if (typeof window !== 'undefined') {
|
||||
|
||||
@@ -7,10 +7,27 @@
|
||||
* commits. Any failure resolves `null` so the caller falls back to a plain
|
||||
* byte upload — delta is an optimization, never a gate.
|
||||
*/
|
||||
import log from 'loglevel';
|
||||
import { getCsrfToken } from '$lib/api/csrf';
|
||||
import { createFileByHash, dedupCheckBatch } from '$lib/api/endpoints/files';
|
||||
import { blake3HexOfFile } from '$lib/vendor/hashWasm';
|
||||
|
||||
// Namespaced logger — level configurable at runtime from the browser
|
||||
// console via `log.getLogger('oxi:upload').setLevel('debug')`, persisted
|
||||
// to `localStorage['loglevel:oxi:upload']`. Default = info so common
|
||||
// phase transitions are visible without extra opt-in; users chasing a
|
||||
// bug flip to `debug` for per-chunk verbose trace without a page reload.
|
||||
const uploadLog = log.getLogger('oxi:upload');
|
||||
uploadLog.setDefaultLevel('info');
|
||||
|
||||
// Short random id — one per upload attempt — so multiple concurrent
|
||||
// files stay distinguishable in the console.
|
||||
function newUploadId(): string {
|
||||
const buf = new Uint8Array(3);
|
||||
crypto.getRandomValues(buf);
|
||||
return Array.from(buf, (b) => b.toString(16).padStart(2, '0')).join('');
|
||||
}
|
||||
|
||||
/** Files smaller than this skip delta: the round-trips cost more than the bytes.
|
||||
* Also the upper bound for client-side whole-file hashing (instant by-hash
|
||||
* uploads) — we never read a file larger than this fully into memory. Files at
|
||||
@@ -49,8 +66,32 @@ interface DoneMsg {
|
||||
type: 'done';
|
||||
status: number;
|
||||
body?: { message?: string; error?: string; still_missing?: unknown };
|
||||
/** Final worker-side counters, sourced from the worker so throttled
|
||||
* progress messages can't undercount on fast dedup-heavy paths. */
|
||||
reusedBytes?: number;
|
||||
uploadedBytes?: number;
|
||||
/** Whole-file BLAKE3 the delta protocol committed — same value the
|
||||
* server stores as `file_blobs.hash`. Correlates a client-side log
|
||||
* line with the resulting server-side blob. */
|
||||
fileHash?: string;
|
||||
}
|
||||
type WorkerMsg = ProgressMsg | FallbackMsg | DoneMsg;
|
||||
/** Worker-emitted log line forwarded to the main-thread `uploadLog` — the
|
||||
* worker can't import loglevel from a static file, so it postMessages
|
||||
* and we relay it through the shared logger. */
|
||||
interface LogMsg {
|
||||
type: 'log';
|
||||
level: 'debug' | 'info' | 'warn' | 'error';
|
||||
msg: string;
|
||||
extra?: Record<string, unknown>;
|
||||
}
|
||||
/** Kept-alive signal the worker emits every 5 s while awaiting a long
|
||||
* fetch (commit, slow chunk PUT). The orchestrator's on-every-message
|
||||
* `armStall()` resets the watchdog just from receiving this — the type
|
||||
* handler below intentionally no-ops so nothing else fires. */
|
||||
interface HeartbeatMsg {
|
||||
type: 'heartbeat';
|
||||
}
|
||||
type WorkerMsg = ProgressMsg | FallbackMsg | DoneMsg | LogMsg | HeartbeatMsg;
|
||||
|
||||
/**
|
||||
* Try to upload `file` through the delta protocol. Resolves `null` whenever
|
||||
@@ -62,21 +103,38 @@ export function tryDeltaUpload(
|
||||
folderId: string | null | undefined,
|
||||
onProgress?: (pct: number) => void
|
||||
): Promise<DeltaUploadAnswer | null> {
|
||||
const id = newUploadId();
|
||||
if (
|
||||
!folderId ||
|
||||
file.size < DELTA_UPLOAD_MIN_SIZE ||
|
||||
usable === false ||
|
||||
typeof Worker === 'undefined'
|
||||
) {
|
||||
// Not a bug — these are the documented skip conditions. Log at
|
||||
// debug so verbose-flag users see why delta was skipped; silent
|
||||
// in the default path (would be noise on every small file).
|
||||
const reason = !folderId
|
||||
? 'no folder id'
|
||||
: file.size < DELTA_UPLOAD_MIN_SIZE
|
||||
? `file below ${DELTA_UPLOAD_MIN_SIZE} B threshold`
|
||||
: usable === false
|
||||
? 'delta previously disabled for this tab'
|
||||
: 'Worker constructor unavailable';
|
||||
uploadLog.debug(`[${id}] delta skipped: ${reason}`, { file: file.name, size: file.size });
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
|
||||
uploadLog.info(`[${id}] delta start`, { file: file.name, size: file.size });
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let worker: Worker;
|
||||
try {
|
||||
worker = new Worker(DELTA_WORKER_URL, { type: 'module' });
|
||||
} catch {
|
||||
} catch (err) {
|
||||
usable = false;
|
||||
uploadLog.warn(`[${id}] delta disabled for this tab: Worker constructor threw`, {
|
||||
error: err instanceof Error ? err.message : String(err)
|
||||
});
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
@@ -86,13 +144,32 @@ export function tryDeltaUpload(
|
||||
let savedBytes = 0;
|
||||
|
||||
let stallTimer: ReturnType<typeof setTimeout>;
|
||||
// Page Visibility listener — pause the stall watchdog while the
|
||||
// tab is hidden. Background tabs get main-thread timer throttling
|
||||
// (Chrome/Firefox: 1 s min tick, ~5 min hidden → timers may pause
|
||||
// entirely) but Web Workers keep running at full speed. Without
|
||||
// this pause, a tab-switch during a large upload would let messages
|
||||
// queue up on the throttled main thread while the watchdog fires
|
||||
// spuriously — poisoning `usable = false` for the rest of the
|
||||
// session even though the worker was healthy the whole time.
|
||||
let visibilityListener: (() => void) | null = null;
|
||||
const settle = (answer: DeltaUploadAnswer | null) => {
|
||||
clearTimeout(timer);
|
||||
clearTimeout(stallTimer);
|
||||
if (visibilityListener) {
|
||||
document.removeEventListener('visibilitychange', visibilityListener);
|
||||
visibilityListener = null;
|
||||
}
|
||||
worker.terminate();
|
||||
resolve(answer);
|
||||
};
|
||||
const timer = setTimeout(() => settle(null), timeoutMs);
|
||||
const timer = setTimeout(() => {
|
||||
uploadLog.error(
|
||||
`[${id}] delta timeout after ${Math.round(timeoutMs / 1000)}s — falling back to direct upload`,
|
||||
{ file: file.name, size: file.size }
|
||||
);
|
||||
settle(null);
|
||||
}, timeoutMs);
|
||||
|
||||
// Liveness watchdog: a healthy worker posts progress sub-second while it
|
||||
// hashes and uploads. If it goes SILENT this long it is wedged (WASM init
|
||||
@@ -100,19 +177,55 @@ export function tryDeltaUpload(
|
||||
// — exactly what freezes a folder upload ~2 min per large file. Disable
|
||||
// delta for this file AND every later one so they fall straight to a plain
|
||||
// upload instead of each burning the full size-scaled delta timeout.
|
||||
//
|
||||
// Long single fetches (commit, slow chunk PUTs) don't emit progress
|
||||
// on their own — the worker sends `{ type: 'heartbeat' }` every 5 s
|
||||
// while awaiting a network request so the watchdog stays fresh.
|
||||
const STALL_MS = 20_000;
|
||||
const armStall = () => {
|
||||
clearTimeout(stallTimer);
|
||||
// Don't count time while the tab is hidden — background throttling
|
||||
// on the main thread breaks the "no message in 20 s = wedged"
|
||||
// premise. When the user comes back, the visibility listener
|
||||
// re-arms.
|
||||
if (typeof document !== 'undefined' && document.hidden) return;
|
||||
stallTimer = setTimeout(() => {
|
||||
usable = false;
|
||||
uploadLog.error(
|
||||
`[${id}] delta worker went silent for ${STALL_MS / 1000}s — disabling delta for this tab (later files this session will go direct)`,
|
||||
{ file: file.name }
|
||||
);
|
||||
settle(null);
|
||||
}, STALL_MS);
|
||||
};
|
||||
if (typeof document !== 'undefined') {
|
||||
visibilityListener = () => {
|
||||
if (document.hidden) clearTimeout(stallTimer);
|
||||
else armStall();
|
||||
};
|
||||
document.addEventListener('visibilitychange', visibilityListener);
|
||||
}
|
||||
armStall();
|
||||
|
||||
worker.onmessage = (event: MessageEvent<WorkerMsg>) => {
|
||||
armStall(); // worker is alive — reset the liveness watchdog
|
||||
const msg = event.data;
|
||||
if (msg.type === 'heartbeat') {
|
||||
// armStall() above already served its purpose — no other work.
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'log') {
|
||||
// Relay worker log through the shared logger so runtime-set
|
||||
// level (via `log.getLogger('oxi:upload').setLevel(...)`)
|
||||
// filters worker output too. Worker id doesn't know the
|
||||
// upload id — we prefix it here for correlation. Skip the
|
||||
// second arg when there's no extras: loglevel would log the
|
||||
// literal `undefined` next to the message otherwise.
|
||||
const line = `[${id}] worker: ${msg.msg}`;
|
||||
if (msg.extra) uploadLog[msg.level](line, msg.extra);
|
||||
else uploadLog[msg.level](line);
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'progress') {
|
||||
savedBytes = msg.reusedBytes;
|
||||
if (onProgress && msg.totalBytes > 0) {
|
||||
@@ -125,33 +238,74 @@ export function tryDeltaUpload(
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'fallback') {
|
||||
uploadLog.warn(`[${id}] worker requested fallback: ${msg.reason ?? 'no reason'}`, {
|
||||
file: file.name
|
||||
});
|
||||
settle(null);
|
||||
return;
|
||||
}
|
||||
if (msg.type === 'done') {
|
||||
if (msg.status === 201 || msg.status === 200) {
|
||||
settle({ ok: true, data: msg.body, savedBytes });
|
||||
// Prefer the worker's authoritative final counter over
|
||||
// the throttled progress-message-derived one — throttling
|
||||
// can hide the reused-bytes update on fast paths.
|
||||
const finalSaved = msg.reusedBytes ?? savedBytes;
|
||||
uploadLog.info(`[${id}] delta done`, {
|
||||
file: file.name,
|
||||
blake3: msg.fileHash,
|
||||
savedBytes: finalSaved,
|
||||
uploadedBytes: msg.uploadedBytes ?? 0
|
||||
});
|
||||
settle({ ok: true, data: msg.body, savedBytes: finalSaved });
|
||||
return;
|
||||
}
|
||||
const errorMsg =
|
||||
msg.body?.message || msg.body?.error || `Delta upload failed (HTTP ${msg.status})`;
|
||||
if (msg.status === 507) {
|
||||
uploadLog.warn(`[${id}] delta hit quota (HTTP 507)`, { file: file.name, errorMsg });
|
||||
settle({ ok: false, isQuotaError: true, errorMsg });
|
||||
return;
|
||||
}
|
||||
if (msg.status === 409 && !msg.body?.still_missing) {
|
||||
uploadLog.warn(`[${id}] delta conflict (HTTP 409)`, { file: file.name, errorMsg });
|
||||
settle({ ok: false, errorMsg });
|
||||
return;
|
||||
}
|
||||
uploadLog.warn(
|
||||
`[${id}] delta done with non-2xx (HTTP ${msg.status}) — falling back to direct upload`,
|
||||
{ file: file.name, errorMsg }
|
||||
);
|
||||
settle(null);
|
||||
}
|
||||
};
|
||||
worker.onerror = () => {
|
||||
worker.onerror = (e) => {
|
||||
usable = false;
|
||||
// Real browsers pass an ErrorEvent; test doubles fire onerror
|
||||
// with no argument. Optional-chain so the no-arg path doesn't
|
||||
// throw on `.message` and mask the real disable-signal.
|
||||
uploadLog.error(`[${id}] delta worker onerror — disabling delta for this tab`, {
|
||||
file: file.name,
|
||||
message: e?.message,
|
||||
filename: e?.filename,
|
||||
lineno: e?.lineno
|
||||
});
|
||||
settle(null);
|
||||
};
|
||||
|
||||
worker.postMessage({ file, folderId, name: file.name, csrfToken: getCsrfToken() || '' });
|
||||
// Runtime-tunable batch size (`window.oxi.UPLOAD_BATCH_BYTES`,
|
||||
// persisted to localStorage). Undefined = worker uses its own
|
||||
// default (8 MiB). Behind Cloudflare Tunnel or other proxies
|
||||
// with tight per-request timeouts, users can lower it via
|
||||
// `oxi.UPLOAD_BATCH_BYTES = 1024 * 1024` so each PUT completes
|
||||
// well inside the proxy's 100 s window on a slow uplink.
|
||||
const uploadBatchBytes = window.oxi?.UPLOAD_BATCH_BYTES;
|
||||
worker.postMessage({
|
||||
file,
|
||||
folderId,
|
||||
name: file.name,
|
||||
csrfToken: getCsrfToken() || '',
|
||||
uploadBatchBytes
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* Interrupted-upload registry — small helper so a page reload during
|
||||
* upload doesn't leave the user without a hint that work was in flight.
|
||||
*
|
||||
* Two coordinated behaviours:
|
||||
*
|
||||
* 1. `beforeunload` warning while any upload is active.
|
||||
* Registered lazily: as soon as an upload starts we install a
|
||||
* page-scope handler that triggers the browser's "Leave site?
|
||||
* Changes may not be saved" prompt on refresh / tab close.
|
||||
* Deliberate leave (user clicks Leave) proceeds normally.
|
||||
*
|
||||
* 2. `sessionStorage`-backed "interrupted uploads" register.
|
||||
* Every `uploadBatch` writes a record while it runs, clears it on
|
||||
* completion. If a reload happens mid-flight, the record survives
|
||||
* into the next page load and `readAndClearInterrupted` surfaces
|
||||
* it so the layout can toast: "Uploads were interrupted — re-drop
|
||||
* the files to resume (already-uploaded chunks reuse)."
|
||||
*
|
||||
* `sessionStorage` (not `localStorage`) on purpose: entries clear
|
||||
* when the tab closes entirely, so a "closed the tab an hour ago"
|
||||
* user isn't nagged. Only a same-tab reload preserves them.
|
||||
*/
|
||||
|
||||
/** Key under which the interrupted-uploads register is stored. */
|
||||
const STORAGE_KEY = 'oxi:upload:interrupted';
|
||||
|
||||
/** One record per active `uploadBatch()` invocation. */
|
||||
export interface InterruptedRecord {
|
||||
/** UI-facing description — filename for singleton uploads, "N files" for batches. */
|
||||
description: string;
|
||||
/** Where the batch was targeted. `null` = drive root. */
|
||||
folderId: string | null;
|
||||
/** UNIX ms — used as the identity key that matches start/finish
|
||||
* calls so multiple concurrent batches don't step on each other. */
|
||||
startedAt: number;
|
||||
}
|
||||
|
||||
// ── Page-scope beforeunload guard ────────────────────────────────────
|
||||
|
||||
let activeBatches = 0;
|
||||
|
||||
function beforeUnloadHandler(e: BeforeUnloadEvent): void {
|
||||
// Spec-compliant trigger for the browser's "Leave site?" dialog.
|
||||
// Modern Chrome/Firefox/Safari all honor preventDefault(). The
|
||||
// browser shows its own confirmation copy — we can't customize it.
|
||||
e.preventDefault();
|
||||
}
|
||||
|
||||
/** Register a live upload batch so the beforeunload guard is active while
|
||||
* it runs. Balanced by `releaseUploadGuard` in the batch's finally. */
|
||||
export function acquireUploadGuard(): void {
|
||||
if (typeof window === 'undefined') return;
|
||||
if (activeBatches === 0) {
|
||||
window.addEventListener('beforeunload', beforeUnloadHandler);
|
||||
}
|
||||
activeBatches++;
|
||||
}
|
||||
|
||||
/** Match to `acquireUploadGuard`; when the last active batch releases,
|
||||
* the beforeunload listener is removed so unrelated navigations don't
|
||||
* trigger the browser's "leave site?" prompt. */
|
||||
export function releaseUploadGuard(): void {
|
||||
if (typeof window === 'undefined') return;
|
||||
activeBatches = Math.max(0, activeBatches - 1);
|
||||
if (activeBatches === 0) {
|
||||
window.removeEventListener('beforeunload', beforeUnloadHandler);
|
||||
}
|
||||
}
|
||||
|
||||
// ── sessionStorage register ──────────────────────────────────────────
|
||||
|
||||
function readAll(): InterruptedRecord[] {
|
||||
if (typeof sessionStorage === 'undefined') return [];
|
||||
try {
|
||||
const raw = sessionStorage.getItem(STORAGE_KEY);
|
||||
if (!raw) return [];
|
||||
const parsed = JSON.parse(raw);
|
||||
return Array.isArray(parsed) ? (parsed as InterruptedRecord[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function writeAll(records: InterruptedRecord[]): void {
|
||||
if (typeof sessionStorage === 'undefined') return;
|
||||
try {
|
||||
if (records.length === 0) sessionStorage.removeItem(STORAGE_KEY);
|
||||
else sessionStorage.setItem(STORAGE_KEY, JSON.stringify(records));
|
||||
} catch {
|
||||
/* quota exceeded / disabled — silent; the register is best-effort. */
|
||||
}
|
||||
}
|
||||
|
||||
/** Add a record when a batch starts. Returns a handle to pass back to
|
||||
* `markUploadFinished` on completion / failure — that way multiple
|
||||
* concurrent batches don't step on each other's entries. */
|
||||
export function markUploadStarted(description: string, folderId: string | null): number {
|
||||
const record: InterruptedRecord = {
|
||||
description,
|
||||
folderId,
|
||||
startedAt: Date.now()
|
||||
};
|
||||
const all = readAll();
|
||||
all.push(record);
|
||||
writeAll(all);
|
||||
return record.startedAt;
|
||||
}
|
||||
|
||||
/** Remove the record when the batch completes (success or failure). Uses
|
||||
* the `markUploadStarted` return value as the identity key. */
|
||||
export function markUploadFinished(startedAt: number): void {
|
||||
const all = readAll();
|
||||
const idx = all.findIndex((r) => r.startedAt === startedAt);
|
||||
if (idx >= 0) {
|
||||
all.splice(idx, 1);
|
||||
writeAll(all);
|
||||
}
|
||||
}
|
||||
|
||||
/** Read and clear the register — called by the layout on mount. Returns
|
||||
* what was there. Empties the register in the same call so a subsequent
|
||||
* refresh doesn't re-notify. */
|
||||
export function readAndClearInterrupted(): InterruptedRecord[] {
|
||||
const all = readAll();
|
||||
writeAll([]);
|
||||
return all;
|
||||
}
|
||||
@@ -15,6 +15,8 @@
|
||||
import { hashUrlToPath } from '$lib/utils/hashRedirect';
|
||||
import { killLegacyServiceWorker } from '$lib/utils/killLegacyServiceWorker';
|
||||
import { getOidcProviders, type OidcProviders } from '$lib/api/endpoints/auth';
|
||||
import { readAndClearInterrupted } from '$lib/upload/interruption';
|
||||
import { t } from '$lib/i18n/index.svelte';
|
||||
|
||||
let { children } = $props();
|
||||
|
||||
@@ -156,6 +158,32 @@
|
||||
// loading state) is already in the DOM behind it.
|
||||
document.getElementById('app-splash')?.remove();
|
||||
|
||||
// If a page reload interrupted one or more uploads mid-flight, the
|
||||
// registry in sessionStorage carries breadcrumbs into the next mount.
|
||||
// Toast a resume hint — already-uploaded chunks reuse via delta's
|
||||
// negotiate stage, so re-dropping the same file is fast, not from
|
||||
// scratch. Reading the register clears it so a subsequent reload
|
||||
// doesn't re-notify.
|
||||
const interrupted = readAndClearInterrupted();
|
||||
if (interrupted.length > 0) {
|
||||
const one = interrupted.length === 1;
|
||||
const msg = one
|
||||
? t(
|
||||
'files.upload_interrupted_one',
|
||||
{ description: interrupted[0].description },
|
||||
`Upload interrupted: ${interrupted[0].description}. Re-drop to resume — already-uploaded chunks are reused.`
|
||||
)
|
||||
: t(
|
||||
'files.upload_interrupted_many',
|
||||
{ n: interrupted.length },
|
||||
`${interrupted.length} uploads interrupted. Re-drop the files to resume — already-uploaded chunks are reused.`
|
||||
);
|
||||
// 10 s dwell — same pattern as the other long-copy toasts
|
||||
// (see profile SSO error). Info kind because the situation is
|
||||
// recoverable and the user only needs to know how to resume.
|
||||
ui.notify(msg, 'info', 10000);
|
||||
}
|
||||
|
||||
// Redirect old `#/...` bookmarks to the new path before anything else.
|
||||
if (typeof location !== 'undefined' && location.hash.startsWith('#/')) {
|
||||
// hashUrlToPath returns a dynamic in-app path string; resolve() is typed
|
||||
|
||||
@@ -30,6 +30,12 @@
|
||||
resolveOwnedHashes,
|
||||
tryDeltaUpload
|
||||
} from '$lib/api/endpoints/deltaUpload';
|
||||
import {
|
||||
acquireUploadGuard,
|
||||
markUploadFinished,
|
||||
markUploadStarted,
|
||||
releaseUploadGuard
|
||||
} from '$lib/upload/interruption';
|
||||
import { addFavorite, removeFavorite } from '$lib/api/endpoints/favorites';
|
||||
import { canEditWithWopi, getEditorUrlWithFallback } from '$lib/api/endpoints/wopi';
|
||||
import { addTracks, createPlaylist, listPlaylists } from '$lib/api/endpoints/music';
|
||||
@@ -710,6 +716,17 @@
|
||||
async function uploadBatch(files: File[]) {
|
||||
if (files.length === 0) return;
|
||||
uploading = true;
|
||||
// Arm the reload-guard + persist a "batch in flight" marker so a
|
||||
// page refresh mid-upload (a) prompts the browser's "Leave site?"
|
||||
// dialog and (b) leaves a breadcrumb the layout picks up on the
|
||||
// next mount → toasts a "uploads were interrupted, re-drop to
|
||||
// resume (chunks reuse)" hint.
|
||||
acquireUploadGuard();
|
||||
const batchDescription =
|
||||
files.length === 1
|
||||
? files[0].name
|
||||
: t('files.n_files_batch', { n: files.length }, `${files.length} files`);
|
||||
const batchHandle = markUploadStarted(batchDescription, currentId);
|
||||
const nid = ui.startProgress(
|
||||
t('files.uploading_n', { done: 0, total: files.length }, `Uploading 0/${files.length} files…`)
|
||||
);
|
||||
@@ -760,6 +777,8 @@
|
||||
ui.finishProgress(nid, errorMessage(err), 'error');
|
||||
} finally {
|
||||
uploading = false;
|
||||
markUploadFinished(batchHandle);
|
||||
releaseUploadGuard();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1539,6 +1558,14 @@
|
||||
async function uploadTree(entries: { file: File; relativePath: string }[]) {
|
||||
if (entries.length === 0) return;
|
||||
uploading = true;
|
||||
// Same reload-guard + interrupted-uploads breadcrumb as uploadBatch —
|
||||
// the browser prompts on refresh, and if the user reloads anyway
|
||||
// the layout picks up the marker on next mount and toasts.
|
||||
acquireUploadGuard();
|
||||
const treeHandle = markUploadStarted(
|
||||
t('files.n_files_batch', { n: entries.length }, `${entries.length} files`),
|
||||
currentId
|
||||
);
|
||||
// Same bell progress notification as uploadBatch, so folder uploads show
|
||||
// live progress + a final result instead of staying silent until the end.
|
||||
const nid = ui.startProgress(
|
||||
@@ -1594,6 +1621,8 @@
|
||||
ui.finishProgress(nid, errorMessage(err), 'error');
|
||||
} finally {
|
||||
uploading = false;
|
||||
markUploadFinished(treeHandle);
|
||||
releaseUploadGuard();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,8 +28,13 @@ const WASM_GLUE_URL = '/vendors/hash-wasm/oxicloud_hash_wasm.js';
|
||||
const SLICE_BYTES = 8 * 1024 * 1024;
|
||||
/** Negotiate after this many freshly hashed chunks (~64 MiB of content). */
|
||||
const NEGOTIATE_BATCH = 256;
|
||||
/** Group missing chunks into PUT bodies of at most this many bytes. */
|
||||
const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024;
|
||||
/** Default target size of a PUT body (grouping multiple chunk frames).
|
||||
* The orchestrator may override this per-upload via the init message's
|
||||
* `uploadBatchBytes` field, sourced from `window.oxi.UPLOAD_BATCH_BYTES`.
|
||||
* Lowering it (say to 1 MiB) helps clients behind proxies with tight
|
||||
* per-request timeouts (Cloudflare Tunnel: 100 s absolute) at the cost
|
||||
* of more requests per file. */
|
||||
const UPLOAD_BATCH_BYTES_DEFAULT = 8 * 1024 * 1024;
|
||||
/** Reclaim consumed queue slots periodically. A head cursor makes dequeue O(1);
|
||||
* compaction bounds the backing array when hashing stays ahead of the network. */
|
||||
const UPLOAD_QUEUE_COMPACT_AT = 4096;
|
||||
@@ -63,10 +68,54 @@ async function loadWasm() {
|
||||
}
|
||||
|
||||
workerScope.onmessage = async (event) => {
|
||||
const { file, folderId, name, csrfToken } = /** @type {{ file: File, folderId: string, name: string, csrfToken: string }} */ (event.data);
|
||||
const { file, folderId, name, csrfToken, uploadBatchBytes } = /** @type {{ file: File, folderId: string, name: string, csrfToken: string, uploadBatchBytes?: number }} */ (event.data);
|
||||
// Per-upload override sourced from `window.oxi.UPLOAD_BATCH_BYTES`
|
||||
// in the main thread. Falls back to the module default (8 MiB).
|
||||
const uploadBatchBytesEff =
|
||||
typeof uploadBatchBytes === 'number' && uploadBatchBytes > 0
|
||||
? uploadBatchBytes
|
||||
: UPLOAD_BATCH_BYTES_DEFAULT;
|
||||
|
||||
/**
|
||||
* Forward a log line to the main-thread orchestrator, which routes it
|
||||
* through the shared `loglevel` logger (namespace `oxi:upload`). Worker
|
||||
* can't `import 'loglevel'` — it's served from /static and isn't
|
||||
* bundler-resolved — so postMessage is the transport.
|
||||
*
|
||||
* @param {'debug'|'info'|'warn'|'error'} level
|
||||
* @param {string} msg
|
||||
* @param {Record<string, unknown>=} extra
|
||||
*/
|
||||
const log = (level, msg, extra) =>
|
||||
workerScope.postMessage({ type: 'log', level, msg, extra });
|
||||
|
||||
/**
|
||||
* Wrap a long-running fetch so the main-thread stall watchdog stays
|
||||
* fresh while the worker is legitimately awaiting the network. The
|
||||
* watchdog resets on every worker → main-thread message; heartbeats
|
||||
* every 5 s keep it from firing during a slow commit or chunk PUT.
|
||||
* Cleanup in `finally` runs regardless of resolve / reject.
|
||||
*
|
||||
* @template T
|
||||
* @param {() => Promise<T>} fn
|
||||
* @returns {Promise<T>}
|
||||
*/
|
||||
const withHeartbeat = async (fn) => {
|
||||
const hb = setInterval(() => workerScope.postMessage({ type: 'heartbeat' }), 5000);
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
clearInterval(hb);
|
||||
}
|
||||
};
|
||||
|
||||
/** @param {string} reason */
|
||||
const fallback = (reason) => workerScope.postMessage({ type: 'fallback', reason });
|
||||
const fallback = (reason) => {
|
||||
log('warn', `worker fallback: ${reason}`);
|
||||
workerScope.postMessage({ type: 'fallback', reason });
|
||||
};
|
||||
|
||||
log('info', `worker start`, { file: name, size: file.size });
|
||||
|
||||
/** @type {Record<string, string>} */
|
||||
const mutHeaders = { 'Content-Type': 'application/json' };
|
||||
@@ -75,6 +124,7 @@ workerScope.onmessage = async (event) => {
|
||||
let wasm;
|
||||
try {
|
||||
wasm = await loadWasm();
|
||||
log('debug', 'wasm loaded');
|
||||
} catch (err) {
|
||||
fallback(`wasm unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return;
|
||||
@@ -139,11 +189,11 @@ workerScope.onmessage = async (event) => {
|
||||
|
||||
const uploadLoop = async () => {
|
||||
while (!failed) {
|
||||
// Take up to UPLOAD_BATCH_BYTES from the queue.
|
||||
// Take up to the effective per-PUT byte cap from the queue.
|
||||
/** @type {WorkerChunk[]} */
|
||||
const batch = [];
|
||||
let bytes = 0;
|
||||
while (uploadHead < uploadQueue.length && bytes < UPLOAD_BATCH_BYTES) {
|
||||
while (uploadHead < uploadQueue.length && bytes < uploadBatchBytesEff) {
|
||||
const c = /** @type {WorkerChunk} */ (uploadQueue[uploadHead]);
|
||||
uploadQueue[uploadHead] = undefined;
|
||||
uploadHead++;
|
||||
@@ -172,23 +222,28 @@ workerScope.onmessage = async (event) => {
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop -- bounded by pool size
|
||||
const wire = await encodeFrames(batch);
|
||||
log('debug', `chunk PUT: ${batch.length} chunks, ${wire.length} bytes`);
|
||||
// eslint-disable-next-line no-await-in-loop -- bounded by pool size
|
||||
const response = await fetch('/api/files/delta/chunks', {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {})
|
||||
},
|
||||
body: wire
|
||||
});
|
||||
const response = await withHeartbeat(() =>
|
||||
fetch('/api/files/delta/chunks', {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {})
|
||||
},
|
||||
body: wire
|
||||
})
|
||||
);
|
||||
if (!response.ok) {
|
||||
failed = `chunk PUT failed (HTTP ${response.status})`;
|
||||
log('error', failed);
|
||||
return;
|
||||
}
|
||||
for (const c of batch) uploadedBytes += c.s;
|
||||
progress();
|
||||
} catch (err) {
|
||||
failed = `chunk PUT failed: ${err instanceof Error ? err.message : String(err)}`;
|
||||
log('error', failed);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -209,13 +264,16 @@ workerScope.onmessage = async (event) => {
|
||||
const run = negotiateTail.then(async () => {
|
||||
if (failed) return;
|
||||
try {
|
||||
const response = await fetch('/api/files/delta/negotiate', {
|
||||
method: 'POST',
|
||||
headers: mutHeaders,
|
||||
body: JSON.stringify({ chunks: fresh.map(({ h, s }) => ({ h, s })) })
|
||||
});
|
||||
const response = await withHeartbeat(() =>
|
||||
fetch('/api/files/delta/negotiate', {
|
||||
method: 'POST',
|
||||
headers: mutHeaders,
|
||||
body: JSON.stringify({ chunks: fresh.map(({ h, s }) => ({ h, s })) })
|
||||
})
|
||||
);
|
||||
if (!response.ok) {
|
||||
failed = failed || `negotiate failed (HTTP ${response.status})`;
|
||||
log('error', `negotiate failed (HTTP ${response.status})`);
|
||||
return;
|
||||
}
|
||||
const missing = new Set(/** @type {{missing: string[]}} */ (await response.json()).missing);
|
||||
@@ -226,10 +284,15 @@ workerScope.onmessage = async (event) => {
|
||||
reusedBytes += c.s;
|
||||
}
|
||||
}
|
||||
log(
|
||||
'info',
|
||||
`negotiate: ${fresh.length} hashes → ${missing.size} missing, ${fresh.length - missing.size} dedup'd`
|
||||
);
|
||||
signalUploaders();
|
||||
progress();
|
||||
} catch (err) {
|
||||
failed = failed || `negotiate failed: ${err instanceof Error ? err.message : String(err)}`;
|
||||
log('error', `negotiate failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
});
|
||||
negotiateTail = run.catch(() => {});
|
||||
@@ -281,6 +344,10 @@ workerScope.onmessage = async (event) => {
|
||||
const fileHash = /** @type {string} */ (fin.file_hash);
|
||||
hashedBytes = file.size;
|
||||
progress(true);
|
||||
// Log the whole-file BLAKE3 immediately so it's visible in the
|
||||
// trace regardless of whether the commit succeeds. Correlates
|
||||
// the client-side view with the server's `file_blobs.hash`.
|
||||
log('info', `hashed — blake3=${fileHash} (${chunks.length} chunks)`);
|
||||
|
||||
// ── Drain: negotiations → uploads → commit ───────────────
|
||||
await Promise.all(negotiations);
|
||||
@@ -300,11 +367,13 @@ workerScope.onmessage = async (event) => {
|
||||
};
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
// eslint-disable-next-line no-await-in-loop -- retry loop
|
||||
const response = await fetch('/api/files/delta/commit', {
|
||||
method: 'POST',
|
||||
headers: mutHeaders,
|
||||
body: JSON.stringify(commitBody)
|
||||
});
|
||||
const response = await withHeartbeat(() =>
|
||||
fetch('/api/files/delta/commit', {
|
||||
method: 'POST',
|
||||
headers: mutHeaders,
|
||||
body: JSON.stringify(commitBody)
|
||||
})
|
||||
);
|
||||
/** @type {any} */
|
||||
let body = null;
|
||||
try {
|
||||
@@ -329,14 +398,16 @@ workerScope.onmessage = async (event) => {
|
||||
}
|
||||
const wire = await encodeFrames(retry);
|
||||
// eslint-disable-next-line no-await-in-loop -- retry loop
|
||||
const put = await fetch('/api/files/delta/chunks', {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {})
|
||||
},
|
||||
body: wire
|
||||
});
|
||||
const put = await withHeartbeat(() =>
|
||||
fetch('/api/files/delta/chunks', {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {})
|
||||
},
|
||||
body: wire
|
||||
})
|
||||
);
|
||||
if (!put.ok) {
|
||||
fallback(`retry chunk PUT failed (HTTP ${put.status})`);
|
||||
return;
|
||||
@@ -349,7 +420,39 @@ workerScope.onmessage = async (event) => {
|
||||
// Conclusive: 201 created, or a real error (quota, name
|
||||
// conflict, validation). The spawner maps it to the uploaders'
|
||||
// UploadAnswer contract.
|
||||
workerScope.postMessage({ type: 'done', status: response.status, body });
|
||||
const ok = response.status >= 200 && response.status < 300;
|
||||
if (ok) {
|
||||
// Human-friendly outcome line — the raw commit line below
|
||||
// still carries the byte counts for anyone who wants them.
|
||||
if (uploadedBytes === 0 && reusedBytes > 0) {
|
||||
log('info', `✅ file already on server — 100% dedup, no bytes transferred (${reusedBytes.toLocaleString()} B reused, blake3=${fileHash})`);
|
||||
} else if (reusedBytes > 0) {
|
||||
const pct = Math.round((100 * reusedBytes) / file.size);
|
||||
log('info', `✅ committed — uploaded ${uploadedBytes.toLocaleString()} B, reused ${reusedBytes.toLocaleString()} B (${pct}% dedup, blake3=${fileHash})`);
|
||||
} else {
|
||||
log('info', `✅ committed — uploaded ${uploadedBytes.toLocaleString()} B (no dedup, blake3=${fileHash})`);
|
||||
}
|
||||
}
|
||||
log(ok ? 'info' : 'warn', `commit HTTP ${response.status}`, {
|
||||
blake3: fileHash,
|
||||
uploadedBytes,
|
||||
reusedBytes,
|
||||
totalBytes: file.size,
|
||||
attempt,
|
||||
});
|
||||
// Include the final counters + file hash on the done envelope
|
||||
// so the orchestrator's summary is accurate even when the last
|
||||
// throttled progress() got skipped (fast dedup-heavy paths
|
||||
// complete under 150ms — progress' throttle window — so
|
||||
// reusedBytes never surfaced via a progress message).
|
||||
workerScope.postMessage({
|
||||
type: 'done',
|
||||
status: response.status,
|
||||
body,
|
||||
reusedBytes,
|
||||
uploadedBytes,
|
||||
fileHash,
|
||||
});
|
||||
return;
|
||||
}
|
||||
} catch (err) {
|
||||
|
||||
Reference in New Issue
Block a user