perf: round 15 — grouped-listing O(N²) rebucket, exif/reseed allocs, tantivy zero-hit snippet skip

Benchmark-gated, same rule as rounds 2–14: every change ships with a
BEFORE/AFTER benchmark and an equivalence/safety gate; an AFTER that doesn't
beat its BEFORE is rolled back. The rule is encoded per harness (GATE FAIL
non-zero exit in the Rust examples, threshold expect() in vitest).

F1 — Grouped listings (trash / recent / favorites / shared-with-me)
re-bucketed the WHOLE accumulated list on every infinite-scroll page.
ResourceSectionsBuilder (new, off the reactive graph) re-buckets only the
fresh page and hands VirtualList the same rows array reference for untouched
buckets. 50×50 (2 500-item) drain: 63 750 → 2 500 bucketOf calls (25.5×),
12.5 → 1.3 ms wall (9.9×); O(N²/page) → O(N). Deep-equal to the full-rebuild
reference at every page for both a contiguous (date) and a non-contiguous
(trash-by-drive) group-by; reference-stability + fallback gated.

B1 — exif Make/Model: the display String was thrown away to allocate the
trimmed copy; display_value_trimmed trims in place (drain + truncate), 2 → 1
alloc per field (8 → 4 allocs/op, 1.26×).

B2 — content-index worker: text_extractor::supports (lowercases MIME +
extension) was called twice per file per drain batch; classify once into a
Vec<bool> and thread it through both uses. 256-file batch: 704 → 353 allocs,
34.5 → 16.7 µs (2.07×).

B3 — tantivy: skip SnippetGenerator::create on a zero-hit content search
(return Ok(vec![]) once top_docs.is_empty()); the per-hit loop was empty.
400-doc index: 1 575.6 → 1 237.2 ns (1.27×), widens with index size.

Harnesses: examples/bench_round15_micro.rs, examples/bench_round15_tantivy.rs,
frontend resourceSections.bench.test.ts; writeup in benches/ROUND15.md. Also
normalizes two round14 bench examples that were committed unformatted
(cargo fmt --all).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012o47jSrtL7xuNGTHXmtiYL

frontend/src/lib/components/ResourceList.svelte

@@ -78,6 +78,7 @@
 	import { formatBytes } from '$lib/utils/format';
 	import { formatDate, iconNameFromClass, fileIconKindClass } from '$lib/utils/display';
 	import { gridColumns } from '$lib/utils/grid';
+	import { ResourceSectionsBuilder } from '$lib/utils/resourceSections';
 	import { fileThumbnailUrl, thumbSizeForView } from '$lib/api/endpoints/files';
 	import {
 		canThumbnailClientSide,
@@ -369,29 +370,24 @@
 	/**
 	 * Partition the visible items into grouped sections when a `bucketOf` is
 	 * active. Server order is preserved within and across buckets (first-seen).
+	 *
+	 * `ResourceSectionsBuilder` re-buckets only the freshly-appended page rather
+	 * than the whole accumulated list, and hands `VirtualList` the same rows
+	 * array reference for every untouched bucket so it skips re-rendering it. An
+	 * infinite-scroll drain of a grouped listing (trash / recent / favorites /
+	 * shared-with-me) collapses from Σ O(N²/page) to O(N) bucketing work
+	 * (benches/ROUND15.md §F1). Held off the reactive graph — a plain
+	 * accumulator keyed by the append cursor, not $state; `sync` is idempotent,
+	 * so if the derive re-fires without an actual append it safely full-rebuilds
+	 * to the same output the pure `buildResourceSections` reference produces.
 	 */
-	const sections = $derived.by(
-		(): Array<{ key: string; label: string; rows: Array<FileItem | FolderItem> }> => {
-			const bucketOf = activeGroup?.bucketOf;
-			if (!bucketOf) return [{ key: '', label: '', rows: visibleItems }];
-			const order: string[] = [];
-			// Transient bucketing map computed inside $derived.by — not reactive state.
-			// eslint-disable-next-line svelte/prefer-svelte-reactivity
-			const map = new Map<string, Array<FileItem | FolderItem>>();
-			for (const item of visibleItems) {
-				const k = bucketOf(item, ctxOf(item.id)) ?? '∅';
-				if (!map.has(k)) {
-					map.set(k, []);
-					order.push(k);
-				}
-				map.get(k)!.push(item);
-			}
-			return order.map((k) => ({
-				key: k,
-				label: activeGroup?.labelOf?.(k) ?? k,
-				rows: map.get(k)!
-			}));
-		}
+	const sectionsBuilder = new ResourceSectionsBuilder<FileItem | FolderItem, ItemContext>();
+	const sections = $derived.by(() =>
+		sectionsBuilder.sync(visibleItems, {
+			bucketOf: activeGroup?.bucketOf,
+			labelOf: activeGroup?.labelOf,
+			ctxOf: (item) => ctxOf(item.id)
+		})
 	);
 	const grouped = $derived(!!activeGroup?.bucketOf);
 

frontend/src/lib/utils/resourceSections.bench.test.ts

@@ -0,0 +1,211 @@
+import { describe, expect, it } from 'vitest';
+import {
+	ResourceSectionsBuilder,
+	buildResourceSections,
+	type SectionGrouping
+} from './resourceSections';
+
+/**
+ * Benchmark gate for the incremental swimlane builder (ResourceSectionsBuilder)
+ * that replaced ResourceList's `sections` `$derived.by`.
+ *
+ * Audit finding (ROUND14 deferred flagship): every grouped listing (trash,
+ * recent, favorites, shared-with-me) pages in via `raw = [...raw, ...page]`,
+ * and `sections` re-bucketed the WHOLE accumulated list on every page — Σ ≈
+ * O(N²/page) `bucketOf` + `ctxOf` calls during an infinite-scroll drain, and a
+ * brand-new rows array for EVERY bucket each page (so VirtualList re-diffed
+ * every swimlane every page). The builder re-buckets only the fresh page and
+ * hands back the same array reference for untouched buckets.
+ *
+ * Gates:
+ *  1. Equivalence — at EVERY page of the drain, the incremental output is
+ *     deep-equal to the verbatim full-rebuild reference (buildResourceSections),
+ *     for a contiguous group-by (date, bucket aligned with order) AND a
+ *     non-contiguous one (trash-by-drive: name-ordered, drive-bucketed); plus
+ *     group-by switch, deletion and the flat pass-through fall back correctly.
+ *  2. Reference stability — untouched buckets keep their exact array reference
+ *     across a page append (the property VirtualList relies on to skip them),
+ *     while a grown bucket gets a fresh one.
+ *  3. Perf — bucketing work collapses from Σ O(N²/page) to O(N) across the
+ *     drain (deterministic `bucketOf`-call count) and wall drops ≥3x.
+ */
+
+interface Item {
+	id: string;
+	name: string;
+	driveId: string;
+	/** ms epoch; descending with index (newest-first, like the server pages). */
+	date: number;
+}
+
+interface Ctx {
+	date: number;
+	driveId: string;
+}
+
+const DAY = 86_400_000;
+
+/** Item `i`: newest-first date, name in a fixed lexical order, round-robin drive. */
+function item(i: number): Item {
+	return {
+		id: `it-${i.toString().padStart(6, '0')}`,
+		// Zero-padded so lexical name order is a stable, well-defined sequence.
+		name: `file-${i.toString().padStart(6, '0')}`,
+		driveId: `drive-${i % 4}`,
+		date: 1_700_000_000_000 - i * (DAY / 2)
+	};
+}
+
+const contextMap = new Map<string, Ctx>();
+function ctxOf(it: Item): Ctx | undefined {
+	let c = contextMap.get(it.id);
+	if (!c) {
+		c = { date: it.date, driveId: it.driveId };
+		contextMap.set(it.id, c);
+	}
+	return c;
+}
+
+/** Month bucket key from a ctx date (contiguous under date order). */
+function monthKey(d: number): string {
+	const dt = new Date(d);
+	return `${dt.getUTCFullYear()}-${`${dt.getUTCMonth() + 1}`.padStart(2, '0')}`;
+}
+
+/** Contiguous group-by: date-ordered pages, date buckets. Counts bucketOf calls. */
+function dateGrouping(counter?: { n: number }): SectionGrouping<Item, Ctx> {
+	return {
+		bucketOf: (_it, ctx) => {
+			if (counter) counter.n++;
+			return ctx ? monthKey(ctx.date) : null;
+		},
+		labelOf: (k) => `📅 ${k}`,
+		ctxOf
+	};
+}
+
+/**
+ * Non-contiguous group-by mirroring trash "by drive": pages arrive in NAME
+ * order but bucket by driveId, so a fresh page sprays items across every
+ * already-emitted drive bucket. Equivalence must still hold.
+ */
+function driveGrouping(counter?: { n: number }): SectionGrouping<Item, Ctx> {
+	return {
+		bucketOf: (_it, ctx) => {
+			if (counter) counter.n++;
+			return ctx ? ctx.driveId : null;
+		},
+		labelOf: (k) => `💾 ${k}`,
+		ctxOf
+	};
+}
+
+const PAGE = 50;
+const PAGES = 50; // 2 500-item drain
+
+describe('incremental resource sections (benchmark gate)', () => {
+	for (const [name, mk] of [
+		['contiguous date buckets', dateGrouping],
+		['non-contiguous drive buckets', driveGrouping]
+	] as const) {
+		it(`stays deep-equal to the full rebuild at every page — ${name}`, () => {
+			const all = Array.from({ length: PAGE * PAGES }, (_, i) => item(i));
+			const builder = new ResourceSectionsBuilder<Item, Ctx>();
+			// ONE stable grouping across the drain — mirrors the component, where
+			// `activeGroup.bucketOf` is a fixed closure from the page's once-defined
+			// `groupBys`. This is what lets the builder take its incremental path,
+			// so this loop genuinely exercises it (not the rebuild fallback).
+			const g = mk();
+			for (let p = 1; p <= PAGES; p++) {
+				const cumulative = all.slice(0, p * PAGE);
+				const incremental = builder.sync(cumulative, g);
+				const reference = buildResourceSections(cumulative, g);
+				expect(incremental, `page ${p}`).toEqual(reference);
+			}
+		});
+	}
+
+	it('keeps untouched bucket arrays reference-stable and refreshes grown ones', () => {
+		const all = Array.from({ length: 600 }, (_, i) => item(i));
+		const builder = new ResourceSectionsBuilder<Item, Ctx>();
+		const g = dateGrouping();
+
+		const first = builder.sync(all.slice(0, 300), g);
+		const refBefore = new Map(first.map((s) => [s.key, s.rows]));
+
+		const second = builder.sync(all.slice(0, 350), g);
+		let stable = 0;
+		let refreshed = 0;
+		for (const s of second) {
+			const prev = refBefore.get(s.key);
+			if (prev === undefined) continue; // brand-new bucket
+			if (prev === s.rows) stable++;
+			else refreshed++;
+		}
+		// Date-ordered append only grows the boundary bucket(s): most earlier
+		// buckets must be handed back by the SAME reference (VirtualList skips
+		// them), and at least one bucket must be refreshed (it grew).
+		expect(stable).toBeGreaterThan(0);
+		expect(refreshed).toBeGreaterThan(0);
+		expect(stable).toBeGreaterThan(refreshed);
+	});
+
+	it('falls back to a correct full rebuild on group-by switch, deletion and flat', () => {
+		const all = Array.from({ length: 600 }, (_, i) => item(i));
+		const builder = new ResourceSectionsBuilder<Item, Ctx>();
+		const byDate = dateGrouping();
+		const byDrive = driveGrouping();
+
+		// Drain a few pages under date grouping, then switch to drive grouping
+		// (a different bucketOf reference → rebuild).
+		builder.sync(all.slice(0, 300), byDate);
+		expect(builder.sync(all.slice(0, 300), byDrive)).toEqual(
+			buildResourceSections(all.slice(0, 300), byDrive)
+		);
+
+		// Deletion under the SAME grouping (list shrinks / prefix changes) →
+		// rebuild via the append check, not a grouping-ref change.
+		const shrunk = all.slice(0, 300).filter((_, i) => i % 7 !== 0);
+		expect(builder.sync(shrunk, byDrive)).toEqual(buildResourceSections(shrunk, byDrive));
+
+		// Flat pass-through (no bucketOf) yields one section and doesn't wedge the
+		// next grouped sync.
+		const flat: SectionGrouping<Item, Ctx> = { ctxOf };
+		const flatOut = builder.sync(shrunk, flat);
+		expect(flatOut).toEqual([{ key: '', label: '', rows: shrunk }]);
+		expect(flatOut[0].rows).toBe(shrunk); // pass-through, no copy
+		expect(builder.sync(shrunk, byDate)).toEqual(buildResourceSections(shrunk, byDate));
+	});
+
+	it('collapses bucketing work from Σ O(N²/page) to O(N) and runs ≥3x faster', () => {
+		const N = PAGE * PAGES;
+		const all = Array.from({ length: N }, (_, i) => item(i));
+
+		// AFTER: incremental — each item is bucketed exactly once across the drain.
+		// ONE stable grouping (fixed bucketOf), exactly as the component supplies.
+		const afterCounter = { n: 0 };
+		const gAfter = dateGrouping(afterCounter);
+		const builder = new ResourceSectionsBuilder<Item, Ctx>();
+		const t1 = performance.now();
+		for (let p = 1; p <= PAGES; p++) builder.sync(all.slice(0, p * PAGE), gAfter);
+		const afterMs = performance.now() - t1;
+
+		// BEFORE: full rebuild per page — re-buckets the whole cumulative list.
+		const beforeCounter = { n: 0 };
+		const gBefore = dateGrouping(beforeCounter);
+		const t0 = performance.now();
+		for (let p = 1; p <= PAGES; p++) buildResourceSections(all.slice(0, p * PAGE), gBefore);
+		const beforeMs = performance.now() - t0;
+
+		console.info(
+			`resource sections ${PAGES}×${PAGE}: before ${beforeCounter.n} bucketOf calls / ${beforeMs.toFixed(1)} ms — after ${afterCounter.n} calls / ${afterMs.toFixed(1)} ms (${(beforeCounter.n / afterCounter.n).toFixed(1)}x fewer calls, ${(beforeMs / afterMs).toFixed(1)}x wall)`
+		);
+
+		// Incremental buckets each item once: exactly N calls.
+		expect(afterCounter.n).toBe(N);
+		// Full rebuild is quadratic: Σ_{p=1..P} p·PAGE.
+		expect(beforeCounter.n).toBe((PAGES * (PAGES + 1) * PAGE) / 2);
+		expect(afterCounter.n).toBeLessThan(beforeCounter.n / 5);
+		expect(afterMs).toBeLessThan(beforeMs / 3);
+	});
+});

frontend/src/lib/utils/resourceSections.ts

@@ -0,0 +1,189 @@
+/**
+ * Incremental swimlane bucketing for `ResourceList`, extracted from the
+ * component so the O(N²) accumulation of its `sections` `$derived` can be
+ * replaced with an append-aware builder (and unit/benchmark-tested off the
+ * Svelte reactive graph).
+ *
+ * `ResourceList` pages its list in via infinite scroll (`raw = [...raw,
+ * ...page]`), and `sections` was `$derived` over the WHOLE accumulated list —
+ * so paging to item N re-buckets everything loaded so far, Σ ≈ O(N²/page)
+ * main-thread work during the scroll (the same class ROUND6 fixed for the
+ * files listing and ROUND14 §F2 fixed for favorites, and PhotoTimeline fixed
+ * for the photos grid).
+ *
+ * Grouped listings sort by the active group's `orderBy`, so a fresh page only
+ * ever extends existing buckets or appends new ones — it never reorders an
+ * already-emitted bucket. {@link ResourceSectionsBuilder} exploits that: an
+ * append re-buckets only the fresh page and hands back the SAME array
+ * reference for every untouched bucket (so `VirtualList`, which diffs its
+ * `items` prop by reference, skips re-rendering it) while emitting a fresh
+ * array for each bucket the page actually grew.
+ *
+ * Correctness does not depend on bucket contiguity: even a group-by whose
+ * `bucketOf` is not monotonic in server order (e.g. trash grouped by drive but
+ * ordered by name) stays byte-for-byte equal to the full rebuild — it just
+ * touches more buckets per page. The pure {@link buildResourceSections} is the
+ * verbatim reference (what the old `sections` derive produced); the benchmark
+ * gate asserts the incremental builder stays deep-equal to it at every page.
+ */
+
+/** One swimlane: a bucket key, its (possibly async-resolved) header label, and its rows. */
+export interface ResourceSection<T> {
+	key: string;
+	label: string;
+	rows: T[];
+}
+
+/**
+ * The grouping inputs the builder needs, mirroring `ResourceList`'s active
+ * `GroupByDef` plus its per-item context accessor. `bucketOf` undefined means
+ * "flat list" (a single unlabelled section). Generic over the item type `T`
+ * and the per-item context envelope `C` so the module stays independent of the
+ * component's concrete types.
+ */
+export interface SectionGrouping<T, C> {
+	/** Map an item + its context to a bucket key; null → the `∅` catch-all bucket. */
+	bucketOf?: (item: T, ctx: C | undefined) => string | null;
+	/** Map a bucket key to its header label; identity when absent. */
+	labelOf?: (key: string) => string;
+	/** Resolve an item's context envelope (e.g. `contextMap.get(item.id)`). */
+	ctxOf: (item: T) => C | undefined;
+}
+
+/** The `∅` catch-all key the old derive used for a null bucket (kept byte-identical). */
+const NULL_BUCKET = '∅';
+
+/**
+ * Verbatim reference: the `ResourceSection[]` the old `sections` `$derived.by`
+ * produced for `items` under `grouping`. Bucket order is first-appearance;
+ * within a bucket, server order is preserved. The benchmark gate holds the
+ * incremental builder equal to this.
+ */
+export function buildResourceSections<T, C>(
+	items: T[],
+	grouping: SectionGrouping<T, C>
+): ResourceSection<T>[] {
+	const bucketOf = grouping.bucketOf;
+	if (!bucketOf) return [{ key: '', label: '', rows: items }];
+	const order: string[] = [];
+	const map = new Map<string, T[]>();
+	for (const item of items) {
+		const k = bucketOf(item, grouping.ctxOf(item)) ?? NULL_BUCKET;
+		let arr = map.get(k);
+		if (arr === undefined) {
+			arr = [];
+			map.set(k, arr);
+			order.push(k);
+		}
+		arr.push(item);
+	}
+	return order.map((k) => ({ key: k, label: grouping.labelOf?.(k) ?? k, rows: map.get(k)! }));
+}
+
+/**
+ * Incremental swimlane builder. Call {@link sync} with the current (already
+ * dotfile-filtered) item list and grouping on every change; it detects the
+ * common case — the list grew by appending a page while the group-by is
+ * unchanged — and re-buckets only the fresh items, reusing every untouched
+ * bucket's array reference so `VirtualList` skips it. Any other change
+ * (group-by switch, deletion, filter toggle, non-append) falls back to a full
+ * rebuild, so the result is always deep-equal to {@link buildResourceSections}.
+ *
+ * Header labels are recomputed on every sync (never cached) because a
+ * group-by's `labelOf` may resolve asynchronously — owner / sharer names
+ * arrive after the rows do, and a cached label would freeze the header at its
+ * fallback. Only the `rows` arrays are reference-stabilised; that is what
+ * `VirtualList` diffs.
+ */
+export class ResourceSectionsBuilder<T, C> {
+	/** Last synced list — the append cursor and the append-detection baseline. */
+	#items: T[] = [];
+	/** Bucket keys in first-appearance order. */
+	#order: string[] = [];
+	/** key → the bucket's rows array (a fresh reference whenever it grows). */
+	#rows = new Map<string, T[]>();
+	/** The `bucketOf` identity of the last grouped sync; a change forces a rebuild. */
+	#bucketOf: SectionGrouping<T, C>['bucketOf'] = undefined;
+	/** False until a grouped sync has populated the accumulation state. */
+	#grouped = false;
+
+	/** Whether `next` extends `prev` (same prefix objects + strictly longer). */
+	#isAppend(prev: T[], next: T[]): boolean {
+		if (next.length <= prev.length) return false;
+		// Prefix identity via the boundary object — O(1); the list is only ever
+		// mutated by appending a page or by replacing it with a filtered copy
+		// (which preserves element identity).
+		return prev.length === 0 || next[prev.length - 1] === prev[prev.length - 1];
+	}
+
+	#rebuild(items: T[], grouping: SectionGrouping<T, C>): void {
+		const bucketOf = grouping.bucketOf!;
+		this.#order = [];
+		this.#rows = new Map();
+		for (const item of items) {
+			const k = bucketOf(item, grouping.ctxOf(item)) ?? NULL_BUCKET;
+			let arr = this.#rows.get(k);
+			if (arr === undefined) {
+				arr = [];
+				this.#rows.set(k, arr);
+				this.#order.push(k);
+			}
+			arr.push(item);
+		}
+		this.#items = items;
+	}
+
+	#extend(items: T[], grouping: SectionGrouping<T, C>): void {
+		const bucketOf = grouping.bucketOf!;
+		const fresh = items.slice(this.#items.length);
+		// Collect the fresh page's items per touched bucket, preserving order and
+		// first-appearance for brand-new buckets. Each touched bucket's array is
+		// then rebuilt exactly once (a fresh reference so VirtualList re-renders
+		// it); untouched buckets keep their existing reference untouched.
+		const freshByKey = new Map<string, T[]>();
+		const newKeys: string[] = [];
+		for (const item of fresh) {
+			const k = bucketOf(item, grouping.ctxOf(item)) ?? NULL_BUCKET;
+			let arr = freshByKey.get(k);
+			if (arr === undefined) {
+				arr = [];
+				freshByKey.set(k, arr);
+				if (!this.#rows.has(k)) newKeys.push(k);
+			}
+			arr.push(item);
+		}
+		for (const [k, add] of freshByKey) {
+			const existing = this.#rows.get(k);
+			this.#rows.set(k, existing ? existing.concat(add) : add);
+		}
+		for (const k of newKeys) this.#order.push(k);
+		this.#items = items;
+	}
+
+	sync(items: T[], grouping: SectionGrouping<T, C>): ResourceSection<T>[] {
+		if (!grouping.bucketOf) {
+			// Flat list: a single pass-through section. Reset accumulation so a
+			// later switch back to a grouped view rebuilds from scratch.
+			this.#grouped = false;
+			this.#bucketOf = undefined;
+			this.#items = items;
+			return [{ key: '', label: '', rows: items }];
+		}
+		if (
+			this.#grouped &&
+			this.#bucketOf === grouping.bucketOf &&
+			this.#isAppend(this.#items, items)
+		) {
+			this.#extend(items, grouping);
+		} else {
+			this.#rebuild(items, grouping);
+		}
+		this.#grouped = true;
+		this.#bucketOf = grouping.bucketOf;
+		return this.#order.map((k) => ({
+			key: k,
+			label: grouping.labelOf?.(k) ?? k,
+			rows: this.#rows.get(k)!
+		}));
+	}
+}