feat: delta/instant upload + frontend UI/UX polish

Bundles the backend+frontend delta-upload (content-dedup) feature with a batch of frontend fixes from this session. Upload / dedup: - Client-hashed delta & instant upload (deltaUpload, hashWasm vendor shim) - Backend dedup batch endpoint (dedup_service, dedup_handler, routes) - session store owned-hash helpers; unit tests + upload-strategy bench Frontend UI/UX: - Colour file-type icons in grid/list (per-type tinted tiles + glyph hue) - Robust thumbnail fallback; PDFs now show their type icon (backend generates no PDF thumbnails) instead of a blank tile - Fix PDF preview: load via a same-origin blob: iframe — the API URL is blocked by the global X-Frame-Options: DENY in the browser's framed PDF viewer, matching the existing CSP `frame-src blob:` design - Groups: localized virtual-group description (no DB schema-note leak), add nav.groups to the 15 missing locales, fix primary-button contrast - Repoint --color-text-light → --color-on-accent (was faint grey on accent) - Nudge the admin role badge off the user-menu header divider Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-20 16:33:08 +02:00
parent f8490ad96e
commit d98e3117b2
35 changed files with 901 additions and 101 deletions
@@ -0,0 +1,87 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+// vi.mock is hoisted; build the spies with vi.hoisted so the factories can use them.
+const { blake3Mock, byHashMock, batchMock } = vi.hoisted(() => ({
+	blake3Mock: vi.fn(),
+	byHashMock: vi.fn(),
+	batchMock: vi.fn()
+}));
+
+vi.mock('$lib/vendor/hashWasm', () => ({ blake3HexOfFile: blake3Mock }));
+vi.mock('$lib/api/endpoints/files', () => ({
+	createFileByHash: byHashMock,
+	dedupCheckBatch: batchMock
+}));
+
+import { DELTA_UPLOAD_MIN_SIZE, instantUploadOwned, resolveOwnedHashes } from './deltaUpload';
+
+const fakeFile = (size: number, name = 'x.bin') => ({ size, name }) as unknown as File;
+const hashOf = (name: string) => name.padEnd(64, '0');
+const MB = 1024 * 1024;
+
+describe('resolveOwnedHashes (batch check)', () => {
+	beforeEach(() => {
+		blake3Mock.mockReset();
+		batchMock.mockReset();
+		blake3Mock.mockImplementation((f: File) => Promise.resolve(hashOf(f.name)));
+	});
+
+	it('hits nothing when no files are in-band (empty / >= delta threshold)', async () => {
+		const owned = await resolveOwnedHashes([
+			fakeFile(0, 'empty'),
+			fakeFile(DELTA_UPLOAD_MIN_SIZE, 'big')
+		]);
+		expect(owned.size).toBe(0);
+		expect(blake3Mock).not.toHaveBeenCalled();
+		expect(batchMock).not.toHaveBeenCalled();
+	});
+
+	it('hashes in-band files and maps only the server-owned subset in ONE batch call', async () => {
+		const a = fakeFile(2 * MB, 'a');
+		const b = fakeFile(3 * MB, 'b');
+		batchMock.mockResolvedValue(new Set([hashOf('a')])); // server owns only "a"
+		const owned = await resolveOwnedHashes([a, b]);
+		expect(batchMock).toHaveBeenCalledTimes(1);
+		expect(batchMock).toHaveBeenCalledWith([hashOf('a'), hashOf('b')]);
+		expect(owned.get(a)).toBe(hashOf('a'));
+		expect(owned.has(b)).toBe(false);
+	});
+
+	it('falls back to an empty map when client-side hashing fails', async () => {
+		blake3Mock.mockRejectedValue(new Error('wasm down'));
+		const owned = await resolveOwnedHashes([fakeFile(2 * MB, 'a')]);
+		expect(owned.size).toBe(0);
+		expect(batchMock).not.toHaveBeenCalled();
+	});
+
+	it('falls back to an empty map when the batch request fails', async () => {
+		batchMock.mockRejectedValue(new Error('network'));
+		const owned = await resolveOwnedHashes([fakeFile(2 * MB, 'a')]);
+		expect(owned.size).toBe(0);
+	});
+});
+
+describe('instantUploadOwned (zero-byte create)', () => {
+	beforeEach(() => byHashMock.mockReset());
+
+	it('reports zero-byte success on 201', async () => {
+		byHashMock.mockResolvedValue({ ok: true, status: 201, data: { id: 'f1' } });
+		const r = await instantUploadOwned('folder', fakeFile(2 * MB, 'a'), hashOf('a'));
+		expect(r).toEqual({ ok: true, data: { id: 'f1' }, savedBytes: 2 * MB });
+		expect(byHashMock).toHaveBeenCalledWith('folder', 'a', hashOf('a'));
+	});
+
+	it('falls back (null) when the blob vanished (404)', async () => {
+		byHashMock.mockResolvedValue({ ok: false, status: 404 });
+		expect(await instantUploadOwned('folder', fakeFile(2 * MB), hashOf('a'))).toBeNull();
+	});
+
+	it('surfaces a quota error on 507', async () => {
+		byHashMock.mockResolvedValue({ ok: false, status: 507 });
+		expect(await instantUploadOwned('folder', fakeFile(2 * MB), hashOf('a'))).toEqual({
+			ok: false,
+			isQuotaError: true,
+			errorMsg: 'Storage quota exceeded'
+		});
+	});
+});
@@ -8,6 +8,8 @@
 * byte upload — delta is an optimization, never a gate.
 */
 import { getCsrfToken } from '$lib/api/csrf';
+import { createFileByHash, dedupCheckBatch } from '$lib/api/endpoints/files';
+import { blake3HexOfFile } from '$lib/vendor/hashWasm';

 /** Files smaller than this skip delta: the round-trips cost more than the bytes. */
 export const DELTA_UPLOAD_MIN_SIZE = 8 * 1024 * 1024;
@@ -128,3 +130,58 @@ export function tryDeltaUpload(
 		worker.postMessage({ file, folderId, name: file.name, csrfToken: getCsrfToken() || '' });
 	});
 }
+
+/**
+ * Create a file from a blob the caller already owns (`POST /api/files/by-hash`)
+ * — zero content bytes cross the wire. `hash` must come from a prior batch
+ * ownership check ([`resolveOwnedHashes`]). Resolves an answer with
+ * `savedBytes = file.size` on success, surfaces a 507 quota error, or resolves
+ * `null` to fall back to a normal upload (e.g. the blob was GC'd between the
+ * check and this create — rare).
+ */
+export async function instantUploadOwned(
+	folderId: string,
+	file: File,
+	hash: string
+): Promise<DeltaUploadAnswer | null> {
+	const res = await createFileByHash(folderId, file.name, hash);
+	if (res.ok) return { ok: true, data: res.data, savedBytes: file.size };
+	if (res.status === 507) {
+		return { ok: false, isQuotaError: true, errorMsg: 'Storage quota exceeded' };
+	}
+	return null;
+}
+
+/**
+ * Resolve which of `files` the server already owns, with a SINGLE batch round
+ * trip (the Dropbox-style "have you got these?" probe). Every file below the
+ * delta threshold is BLAKE3-hashed locally, the whole hash set is sent to
+ * `/api/dedup/check-batch`, and the owned subset is mapped back to `file → hash`
+ * so callers can instant-upload those (zero bytes) and upload the rest normally.
+ *
+ * Excludes empty files and files `>= DELTA_UPLOAD_MIN_SIZE` (the delta protocol
+ * dedups those itself). Resolves an empty map on any failure — hashing
+ * unavailable, request error — so uploads always proceed.
+ */
+export async function resolveOwnedHashes(files: File[]): Promise<Map<File, string>> {
+	const inBand = files.filter((f) => f.size > 0 && f.size < DELTA_UPLOAD_MIN_SIZE);
+	if (inBand.length === 0) return new Map();
+
+	const hashByFile = new Map<File, string>();
+	try {
+		for (const f of inBand) hashByFile.set(f, await blake3HexOfFile(f));
+	} catch {
+		return new Map(); // WASM/hashing unavailable → skip instant uploads
+	}
+
+	let owned: Set<string>;
+	try {
+		owned = await dedupCheckBatch([...new Set(hashByFile.values())]);
+	} catch {
+		return new Map();
+	}
+
+	const result = new Map<File, string>();
+	for (const [f, h] of hashByFile) if (owned.has(h)) result.set(f, h);
+	return result;
+}
@@ -4,6 +4,46 @@ import { getCsrfHeaders } from '$lib/api/csrf';

 const JSON_HEADERS = { 'Content-Type': 'application/json' };

+/**
+ * Instant upload: materialise a file from a blob the caller **already owns**,
+ * by its whole-file BLAKE3 — zero content bytes cross the wire. Returns the HTTP
+ * status so the caller can fall back to a plain upload on 404 (hash not owned).
+ * Scoped to the caller's own content server-side (no cross-user probing).
+ */
+export async function createFileByHash(
+	folderId: string,
+	name: string,
+	hash: string
+): Promise<{ ok: boolean; status: number; data?: unknown }> {
+	const res = await apiFetch('/api/files/by-hash', {
+		method: 'POST',
+		credentials: 'same-origin',
+		headers: { ...JSON_HEADERS, ...getCsrfHeaders() },
+		body: JSON.stringify({ name, folder_id: folderId, hash })
+	});
+	const data = res.ok ? await res.json().catch(() => undefined) : undefined;
+	return { ok: res.ok, status: res.status, data };
+}
+
+/**
+ * Batch dedup check: given candidate whole-file BLAKE3 hashes, return the set
+ * the caller **already owns** — in a single round trip. Drives instant uploads:
+ * a file whose hash is in the set can be created with zero content bytes.
+ * Resolves an empty set on any failure, so the caller just uploads everything.
+ */
+export async function dedupCheckBatch(hashes: string[]): Promise<Set<string>> {
+	if (hashes.length === 0) return new Set();
+	const res = await apiFetch('/api/dedup/check-batch', {
+		method: 'POST',
+		credentials: 'same-origin',
+		headers: { ...JSON_HEADERS, ...getCsrfHeaders() },
+		body: JSON.stringify({ hashes })
+	});
+	if (!res.ok) return new Set();
+	const data = (await res.json().catch(() => null)) as { owned?: string[] } | null;
+	return new Set(data?.owned ?? []);
+}
+
 export async function uploadFile(folderId: string | null, file: File): Promise<void> {
 	const form = new FormData();
 	if (folderId) form.append('folder_id', folderId);
@@ -22,6 +22,17 @@ const VIRTUAL_NAME_KEYS: Record<string, string> = {
 	[INTERNAL_GROUP_ID]: 'groups.virtual_internal_name'
 };

+/**
+ * Map of well-known virtual-group UUIDs → i18n key for a human-readable
+ * description. Virtual groups are server-seeded and their `description` column
+ * holds developer/schema notes (e.g. the Internal group's "…no rows in
+ * subject_group_members."), which must never reach end users — so virtual
+ * groups display a localized blurb instead of their raw `description`.
+ */
+const VIRTUAL_DESC_KEYS: Record<string, string> = {
+	[INTERNAL_GROUP_ID]: 'groups.virtual_internal_explanation'
+};
+
 export interface GroupItem {
 	id: string;
 	name: string;
@@ -87,6 +98,20 @@ export function groupDisplayName(group: GroupItem): string {
 	return group.name;
 }

+/**
+ * Human-readable description for a group row. Virtual groups render a localized
+ * blurb (via the well-known UUID mapping) so their internal DB schema notes
+ * never leak to the UI; a virtual group without a mapped key shows nothing.
+ * User-defined groups display their raw `description` verbatim.
+ */
+export function groupDescription(group: GroupItem): string | null {
+	if (group.is_virtual) {
+		const key = VIRTUAL_DESC_KEYS[group.id];
+		return key ? t(key, group.name) : null;
+	}
+	return group.description ?? null;
+}
+
 /**
 * Pick the icon registry name for a group avatar. Virtual (system-wide)
 * groups use `people-roof`; user-defined groups use `user-group`. Ported from
@@ -0,0 +1,129 @@
+/**
+ * Benchmark: upload-dedup strategies compared.
+ *
+ *   BASELINE  — no instant upload: every file's bytes are sent.
+ *   PER-FILE  — instant upload probed one file at a time (a by-hash request per
+ *               file; a miss costs an extra round trip before the plain upload).
+ *   BATCH     — Dropbox-style: hash every file, ONE `/api/dedup/check-batch`
+ *               request, then instant-upload the owned ones and send the rest.
+ *
+ * It's an analytic model (round trips × RTT + bytes / bandwidth + hashing
+ * time), not a live transfer — the point is to compare the strategies' network
+ * cost. Run it to see the table:
+ *   npm run test:unit -- uploadStrategies
+ */
+import { describe, expect, it } from 'vitest';
+
+interface Cost {
+	roundTrips: number;
+	mbSent: number;
+	mbHashed: number;
+	seconds: number;
+}
+interface Scenario {
+	name: string;
+	files: number;
+	sizeMB: number;
+	ownedFrac: number;
+	rttMs: number;
+	mbps: number;
+}
+
+/** BLAKE3 + file read throughput on a typical client (MB/s). */
+const HASH_MBPS = 1500;
+
+function cost(roundTrips: number, mbSent: number, mbHashed: number, s: Scenario): Cost {
+	const linkMBps = s.mbps / 8;
+	const seconds = roundTrips * (s.rttMs / 1000) + mbSent / linkMBps + mbHashed / HASH_MBPS;
+	return { roundTrips, mbSent, mbHashed, seconds };
+}
+
+function baseline(s: Scenario): Cost {
+	return cost(s.files, s.files * s.sizeMB, 0, s);
+}
+function perFile(s: Scenario): Cost {
+	const owned = Math.round(s.files * s.ownedFrac);
+	const miss = s.files - owned;
+	// owned → 1 by-hash create; miss → by-hash 404 + plain upload. Every file hashed.
+	return cost(owned + miss * 2, miss * s.sizeMB, s.files * s.sizeMB, s);
+}
+function batch(s: Scenario): Cost {
+	const owned = Math.round(s.files * s.ownedFrac);
+	const miss = s.files - owned;
+	// 1 batch check + owned creates + miss uploads. Every file hashed.
+	return cost(1 + owned + miss, miss * s.sizeMB, s.files * s.sizeMB, s);
+}
+
+const SCENARIOS: Scenario[] = [
+	{
+		name: '200×4MB · 50% re-upload · home (40ms/50Mbps)',
+		files: 200,
+		sizeMB: 4,
+		ownedFrac: 0.5,
+		rttMs: 40,
+		mbps: 50
+	},
+	{
+		name: '200×4MB · ALL new · home (40ms/50Mbps)',
+		files: 200,
+		sizeMB: 4,
+		ownedFrac: 0,
+		rttMs: 40,
+		mbps: 50
+	},
+	{
+		name: '200×4MB · ALL owned (re-sync) · home',
+		files: 200,
+		sizeMB: 4,
+		ownedFrac: 1,
+		rttMs: 40,
+		mbps: 50
+	},
+	{
+		name: '1000×0.5MB · 30% owned · WAN (80ms/100Mbps)',
+		files: 1000,
+		sizeMB: 0.5,
+		ownedFrac: 0.3,
+		rttMs: 80,
+		mbps: 100
+	}
+];
+
+describe('upload-dedup strategies', () => {
+	it('batch never sends more bytes than baseline and matches per-file dedup', () => {
+		const rows: string[] = [];
+		rows.push('');
+		rows.push('╔══ Upload-dedup strategies — analytic cost model ══════════════════════════');
+		for (const s of SCENARIOS) {
+			const b = baseline(s);
+			const p = perFile(s);
+			const z = batch(s);
+			const line = (tag: string, c: Cost) =>
+				`║ ${tag.padEnd(9)} │ RT ${String(c.roundTrips).padStart(4)} │ sent ${c.mbSent
+					.toFixed(0)
+					.padStart(4)} MB │ ~${c.seconds.toFixed(1).padStart(6)} s`;
+			rows.push(`╟─ ${s.name}`);
+			rows.push(line('baseline', b));
+			rows.push(line('per-file', p));
+			rows.push(line('BATCH', z));
+			const vsBase = (1 - z.seconds / b.seconds) * 100;
+			const rtVsPerFile = p.roundTrips - z.roundTrips;
+			rows.push(
+				`║   → BATCH: ${vsBase.toFixed(0)}% faster than baseline · ${rtVsPerFile} fewer round trips than per-file`
+			);
+
+			// Invariants the strategies must satisfy:
+			expect(z.mbSent).toBe(p.mbSent); // batch and per-file dedup identically
+			expect(z.mbSent).toBeLessThanOrEqual(b.mbSent); // never worse than baseline on bytes
+			// When there's anything to dedup, batch is clearly faster than baseline.
+			// (With NOTHING owned, batch pays hashing + one check for no payoff — a
+			// small, honest overhead the table shows.)
+			if (s.ownedFrac > 0) expect(z.seconds).toBeLessThan(b.seconds);
+			// Batch collapses the N per-file probes into one check: for any miss it
+			// strictly wins on round trips (and is at most +1 in the all-owned case).
+			if (s.ownedFrac < 1) expect(z.roundTrips).toBeLessThan(p.roundTrips);
+		}
+		rows.push('╚═══════════════════════════════════════════════════════════════════════════');
+		console.log(rows.join('\n'));
+	});
+});