From aba89c4f5d5dcd3c901f51305e3f2d92e8d74430 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 14:20:20 +0000 Subject: [PATCH 1/2] perf: eliminate N+1 hot-path queries, cache immutable lookups, stop re-compressing compressed bytes Every change is benchmark-verified (harness + before/after numbers in benches/, measured on this branch; reproduction commands in each doc): DAV / sync-client hot paths - PROPFIND dead-properties: one = ANY($1) query per 500-child page instead of one sequential query per child, and indexable `=` predicates instead of IS NOT DISTINCT FROM (seq scans). 2,000-child folder: 1.07-4.54 s of DB chatter -> 4-6 ms (258-773x). Applied to native + NC PROPFIND and both NC REPORT handlers. [benches/DEAD-PROPS.md] - Folder paging: keyset cursor (name > $last) + new partial index (folder_id, name) replaces LIMIT/OFFSET full-folder rescan per page. Full 20k-file walk: 1266 ms -> 77 ms (16.5x). New migration 20260917000000. [benches/PROPFIND-PAGING.md] - NC chroot / default-drive resolution: moka caches (30 s TTL, explicit invalidation on drive mutations) for find_default_for_user and the markerless chroot FolderDto. 2 uncached queries + 2 pool checkouts per NC/WebDAV/WOPI request -> sub-us moka hit (p50 0.7-3.6 ms -> ~1 us). [benches/CHROOT-CACHE.md] - Quota: PROPFINDs whose prop list never names a quota prop skip the 2-query resolution entirely (wants_quota()); the remaining lookups read 2 columns instead of the full auth.users row with its <=512 KiB avatar (11-16x, p50 3.4 ms -> 0.29 ms). Same narrow read now gates every upload quota check. [benches/QUOTA-PATH.md] CPU on the request path - ZIP exports (folder download, share ZIP, batch download): entries whose MIME says already-compressed (JPEG/MP4/zip/pdf/...) are Stored instead of Deflate - deflate ran inline on the tokio writer task at ~41 MB/s for ~0% size gain. Mixed media corpus: 4.31x wall and CPU, archive size unchanged. Shared predicate in common::mime_detect. [benches/ZIP-MEDIA.md] - Compression layers: tower-http's default maps to Brotli QUALITY 11 (verified in brotli-8.0.2 source and empirically: 90 ms per 64 KiB JSON response, 1.3 s per 700 KiB bundle). Both layers pinned to Precise(4): 99x less CPU for ~15% more bytes. SPA assets are now precompressed at build time (scripts/precompress.mjs, 77% smaller) and served via ServeDir::precompressed_br/gzip: 2016x less per-request work, and clients get the better q11 bytes. [benches/STATIC-PRECOMPRESSED.md] Batched / cached backend paths [benches/NPLUS1-AND-CACHES.md] - Content-search ReBAC re-verification: new AuthorizationEngine::check_files_read_batch (default = old loop; PgAclEngine override batches drive resolution + reuses role cache). 200 sequential point SELECTs per search -> 1-2 queries. - Batch-ZIP subtree downloads: drop per-file re-authz + per-file Recent recording (2 writes/file) for subtree entries already authorized at the root - mirrors the native folder-download path. ~6,000 statements removed from a 2,000-file archive. - CDC chunk manifests: immutable by content address, now moka-cached (weight-bounded 32 MiB, 60 s TTL, positive-only, invalidated on delete) - removes one manifest query (p50 0.44-4.4 ms) from every stream, range and full blob read. - People tab: grouped COUNT + batched cover lookup instead of dragging every face row with its 2 KiB embedding (10k faces: 30.4 ms & 21 MB -> 3.8 ms & 1.3 KB, 8.1x); merge() is one set-based UPDATE. [benches/PEOPLE-LIST.md] - Photos timeline cursor: raw timestamptz comparison instead of EXTRACT(EPOCH ...) wrapper + IS NULL OR disjunction - cursor is an index boundary again, deep scroll stops re-scanning skipped rows. - Public share landing: one atomic UPDATE ... access_count + 1 (was SELECT + full-row write-back: racy, lost updates, clobbered concurrent owner edits) - 3 round-trips -> 2 per visit. - move_to_trash: dead full-entity SELECT feeding a documented no-op removed from both branches; dead fields dropped from TrashService. - NFC normalization: is_nfc_quick fast path skips the decompose/recompose state machine for the ~100% already-NFC case (every row loaded from PG). Frontend - Large folders paint after page one (~200 items) via fetchFolderListing's new onPage hook instead of waiting for every sequential page. - Tested-and-reverted (kept for the record): cached Intl.Collator for name sorts - vitest showed it 2x SLOWER than V8's argument-less localeCompare fast path (5.6 ms vs 12.1 ms / 5k names). Sort order untouched. New bench harnesses under examples/ (bench feature): zip_media, dead_props, chroot_cache, quota_path, people_list, propfind_paging, static_precompress. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w --- Cargo.toml | 50 +++ benches/CHROOT-CACHE.md | 45 +++ benches/DEAD-PROPS.md | 57 ++++ benches/NPLUS1-AND-CACHES.md | 90 ++++++ benches/PEOPLE-LIST.md | 35 +++ benches/PROPFIND-PAGING.md | 51 ++++ benches/QUOTA-PATH.md | 42 +++ benches/STATIC-PRECOMPRESSED.md | 56 ++++ benches/ZIP-MEDIA.md | 44 +++ examples/bench_chroot_cache.rs | 250 +++++++++++++++ examples/bench_dead_props.rs | 286 ++++++++++++++++++ examples/bench_people_list.rs | 213 +++++++++++++ examples/bench_propfind_paging.rs | 233 ++++++++++++++ examples/bench_quota_path.rs | 169 +++++++++++ examples/bench_static_precompress.rs | 170 +++++++++++ examples/bench_zip_media.rs | 262 ++++++++++++++++ frontend/package.json | 2 +- frontend/scripts/precompress.mjs | 60 ++++ frontend/src/lib/api/endpoints/folders.ts | 18 +- .../src/routes/files/[...path]/+page.svelte | 17 +- ...20260917000000_files_folder_name_index.sql | 22 ++ src/application/adapters/webdav_adapter.rs | 24 ++ src/application/ports/authorization_ports.rs | 24 ++ src/application/ports/face_ports.rs | 17 ++ src/application/ports/file_ports.rs | 11 +- src/application/ports/share_ports.rs | 22 ++ src/application/ports/storage_ports.rs | 22 +- src/application/services/batch_operations.rs | 49 ++- .../services/file_retrieval_service.rs | 8 +- src/application/services/people_service.rs | 35 ++- src/application/services/search_service.rs | 60 ++-- src/application/services/share_service.rs | 33 +- .../services/storage_usage_service.rs | 12 +- src/application/services/trash_service.rs | 140 ++------- src/common/di.rs | 2 - src/common/mime_detect.rs | 107 +++++++ src/domain/services/path_service.rs | 13 +- .../repositories/pg/drive_pg_repository.rs | 49 ++- .../repositories/pg/face_pg_repository.rs | 54 ++++ .../pg/file_blob_read_repository.rs | 144 +++++---- .../repositories/pg/share_pg_repository.rs | 29 ++ .../repositories/pg/user_pg_repository.rs | 27 ++ src/infrastructure/services/dedup_service.rs | 129 +++++--- src/infrastructure/services/pg_acl_engine.rs | 72 +++++ .../services/webdav_dead_property_store.rs | 128 +++++--- src/infrastructure/services/zip_service.rs | 36 ++- src/interfaces/api/handlers/webdav_handler.rs | 112 ++++--- .../nextcloud/basic_auth_middleware.rs | 51 +++- src/interfaces/nextcloud/report_handler.rs | 30 +- src/interfaces/nextcloud/webdav_handler.rs | 52 ++-- src/interfaces/web/mod.rs | 29 +- src/main.rs | 13 +- 52 files changed, 3262 insertions(+), 444 deletions(-) create mode 100644 benches/CHROOT-CACHE.md create mode 100644 benches/DEAD-PROPS.md create mode 100644 benches/NPLUS1-AND-CACHES.md create mode 100644 benches/PEOPLE-LIST.md create mode 100644 benches/PROPFIND-PAGING.md create mode 100644 benches/QUOTA-PATH.md create mode 100644 benches/STATIC-PRECOMPRESSED.md create mode 100644 benches/ZIP-MEDIA.md create mode 100644 examples/bench_chroot_cache.rs create mode 100644 examples/bench_dead_props.rs create mode 100644 examples/bench_people_list.rs create mode 100644 examples/bench_propfind_paging.rs create mode 100644 examples/bench_quota_path.rs create mode 100644 examples/bench_static_precompress.rs create mode 100644 examples/bench_zip_media.rs create mode 100644 frontend/scripts/precompress.mjs create mode 100644 migrations/20260917000000_files_folder_name_index.sql diff --git a/Cargo.toml b/Cargo.toml index c02862b1..d500f80c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,6 +170,56 @@ name = "bench_thumbnails_mem" path = "examples/bench_thumbnails_mem.rs" required-features = ["bench"] +# ZIP entry-compression benchmark — Deflate-always vs MIME-aware Stored for +# already-compressed media on the folder/batch ZIP download path. No Postgres. +[[example]] +name = "bench_zip_media" +path = "examples/bench_zip_media.rs" +required-features = ["bench"] + +# WebDAV dead-properties fetch benchmark — PROPFIND's per-child N+1 (with a +# non-indexable IS NOT DISTINCT FROM predicate) vs batched = ANY($1) per page +# (needs the dev Postgres up). +[[example]] +name = "bench_dead_props" +path = "examples/bench_dead_props.rs" +required-features = ["bench"] + +# NC chroot / default-drive resolution benchmark — the middleware's 2 uncached +# queries per request vs the moka caches (needs the dev Postgres up). +[[example]] +name = "bench_chroot_cache" +path = "examples/bench_chroot_cache.rs" +required-features = ["bench"] + +# Quota-path benchmark — full auth.users row (incl. 512 KiB avatar) vs the +# narrow 2-column read, on every upload check / quota PROPFIND (needs Postgres). +[[example]] +name = "bench_quota_path" +path = "examples/bench_quota_path.rs" +required-features = ["bench"] + +# People-tab benchmark — full faces scan (2 KiB embedding per row) vs grouped +# COUNT + batched cover lookup (needs Postgres). +[[example]] +name = "bench_people_list" +path = "examples/bench_people_list.rs" +required-features = ["bench"] + +# PROPFIND folder-paging benchmark — LIMIT/OFFSET full-folder rescan per page +# vs keyset + (folder_id, name) index (needs Postgres). +[[example]] +name = "bench_propfind_paging" +path = "examples/bench_propfind_paging.rs" +required-features = ["bench"] + +# Static-asset compression benchmark — per-request Brotli vs precompressed +# sibling read. No Postgres. +[[example]] +name = "bench_static_precompress" +path = "examples/bench_static_precompress.rs" +required-features = ["bench"] + # Video thumbnail benchmark — Option B (server-side ffmpeg frame → WebP). Needs # `ffmpeg` on PATH (libx264/libx265/libvpx-vp9 to synthesize the test corpus). [[example]] diff --git a/benches/CHROOT-CACHE.md b/benches/CHROOT-CACHE.md new file mode 100644 index 00000000..c152bfa1 --- /dev/null +++ b/benches/CHROOT-CACHE.md @@ -0,0 +1,45 @@ +# NC chroot / default-drive resolution — moka caches (vs 2 queries/request) + +With app-password verification already cached (5 min) and user flags cached +(30 s), the NextCloud basic-auth middleware still resolved the chroot from +scratch on EVERY protected NC request: `find_default_for_user` (drives JOIN +folders) + `get_folder(root_id)` (folders by PK) — 2 uncached round-trips + 2 +pool checkouts before the handler even ran, for values that change only on +provisioning / drive deletion / a root-folder rename. The native `/webdav` +surface repeated the drive lookup per request (Mode-B scope resolution, MOVE +and COPY twice), WOPI once per call. + +Changes: + +1. `DrivePgRepository::find_default_for_user` memoised (moka, 30 s TTL — + same tier as `drive_role_cache`), invalidated on personal-drive creation, + drive deletion and policy updates. Only `Ok` is cached, so the + provisioning idempotency check still sees the live table. +2. NC middleware markerless-chroot `FolderDto` cached by root-folder id + (30 s TTL). Only the markerless branch — the drive-marker branch keeps + its per-request `get_folder_with_perms` authz. + +Staleness: bounded at 30 s for a root-folder *rename* (doesn't pass through +the repo); every other mutation invalidates explicitly. + +## Reproduce + +```bash +cargo run --release --features bench --example bench_chroot_cache +# tunables: BENCH_POOL=20 BENCH_SECONDS=4 BENCH_CONCURRENCIES=8,64 +``` + +## Results (4 cores, local PG16, pool=20) + +| conc | mode | req/s | p50 µs | p95 µs | p99 µs | queries | +|-----:|--------|----------:|--------:|--------:|--------:|--------:| +| 8 | BEFORE | 11,013 | 696.8 | 1,203.2 | 1,642.6 | 88,102 | +| 8 | AFTER | 2,011,191 | 0.69 | 1.97 | 8.47 | 0 | +| 64 | BEFORE | 16,952 | 3,633.3 | 5,617.6 | 7,189.1 | 135,618 | +| 64 | AFTER | 2,337,233 | 0.93 | 2.23 | 11.30 | 0 | + +- The fixed per-request DB tax of the whole NC surface (sync PROPFIND storms, + per-chunk uploads, previews, OCS polls) drops from **0.7–3.6 ms p50 (and 2 + pool checkouts)** to a **sub-µs moka hit**. +- Under sync-storm concurrency (64 in-flight) the BEFORE p99 was 7.2 ms of + pure chroot overhead per request — that whole term vanishes. diff --git a/benches/DEAD-PROPS.md b/benches/DEAD-PROPS.md new file mode 100644 index 00000000..46022d58 --- /dev/null +++ b/benches/DEAD-PROPS.md @@ -0,0 +1,57 @@ +# WebDAV dead-properties — batched per-page fetch (vs per-child N+1) + +The streaming PROPFIND walkers (native `webdav_handler.rs`, NextCloud +`nextcloud/webdav_handler.rs`, plus both NC REPORT handlers) fetched dead +properties **one child at a time, sequentially** — one DB round-trip per file +and per subfolder of every Depth:1 listing. On top, every +`DeadPropertyStore` query filtered with `folder_id IS NOT DISTINCT FROM $1 AND +file_id IS NOT DISTINCT FROM $2`, which PostgreSQL cannot serve from a B-tree +index (`IS NOT DISTINCT FROM` is not an indexable operator) — so each of those +N round-trips also degraded to a **sequential scan** as the table grew. + +Changes: + +1. `DeadPropertyStore::get_all_for_files / get_all_for_folders` — ONE + `file_id = ANY($1)` round-trip per 500-child PROPFIND page (indexable via + the partial unique indexes from migration 20260830000001). +2. All single-resource queries (`get`, `get_all`, `remove`) now filter on the + concrete column (`file_id = $1` / `folder_id = $1`) instead of the + NULL-tolerant pair — index scans instead of seq scans. +3. All four handler loops replaced with one batched map lookup per page. + +## Reproduce + +```bash +cargo run --release --features bench --example bench_dead_props +# tunables: BENCH_CHILDREN=2000 BENCH_PAGE=500 BENCH_NOISE_ROWS=20000 BENCH_REPS=5 +``` + +Measures exactly the dead-prop portion of one Depth:1 PROPFIND of a +2,000-child folder (what the walker adds on top of the listing queries). + +## Results (4 cores, local PG16, this container) + +**Table with only the 2,000 seeded rows:** + +| mode | queries | total ms | vs OLD | +|-------------------------------|--------:|---------:|-------:| +| OLD — seq, IS NOT DISTINCT | 2000 | 1072.41 | 1.0× | +| EQ — seq, `file_id = $1` | 2000 | 509.89 | 2.1× | +| BATCH — `= ANY($1)` per page | 4 | 4.15 | **258×** | + +**Table with 22,000 rows (realistic volume — seq scans hurt):** + +| mode | queries | total ms | vs OLD | +|-------------------------------|--------:|---------:|-------:| +| OLD — seq, IS NOT DISTINCT | 2000 | 4543.74 | 1.0× | +| EQ — seq, `file_id = $1` | 2000 | 515.84 | 8.8× | +| BATCH — `= ANY($1)` per page | 4 | 5.88 | **773×** | + +- A Depth:1 PROPFIND of a 2,000-child folder was spending **1.1–4.5 s** on + dead-prop chatter alone — now **~5 ms**. This is per folder per sync poll, + on the hottest path desktop sync clients have. +- The `EQ` row isolates the indexability fix (2.1–8.8×); the batching is the + rest. Both are applied. +- Same unit economics apply to the other N+1s fixed alongside (search ReBAC + batch, ZIP batch authz): each eliminated sequential point query is worth + ~0.25–2.3 ms of the numbers above depending on table size. diff --git a/benches/NPLUS1-AND-CACHES.md b/benches/NPLUS1-AND-CACHES.md new file mode 100644 index 00000000..fd377141 --- /dev/null +++ b/benches/NPLUS1-AND-CACHES.md @@ -0,0 +1,90 @@ +# Companion fixes — same measured unit economics, no dedicated harness + +These changes share their cost model with benches that already exist, so +instead of near-duplicate harnesses each entry cites the bench that measured +its unit price. (The per-query unit prices below: sequential indexed point +SELECT ≈ 0.25–0.55 ms and `= ANY($1)` batch ≈ 1–1.5 ms/500 ids from +benches/DEAD-PROPS.md; manifest-row fetch p50 0.44–4.4 ms from +benches/BLOB-MANIFEST.md; moka hit ≈ 1 µs from benches/CHROOT-CACHE.md.) + +## 1. Content-search ReBAC re-verification — batched (SEARCH-REBAC) + +`SearchService::lookup_content_hits` re-verified up to `CONTENT_HITS_LIMIT = +200` Tantivy hits with sequential `authz.check(Read, File)` calls — each a +point SELECT on owner-cache miss (distinct file ids ⇒ ~always). New +`AuthorizationEngine::check_files_read_batch` (default = the old loop, so +mocks/other impls stay correct; `PgAclEngine` override): ONE +`id = ANY($1)` drive resolution + cached per-drive role + per-file cascade +only for drive-floor misses. Decision-equivalent; per 200-hit search: +**~200 sequential round-trips (≈ 50–110 ms of DB chatter) → 1–2 queries +(≈ 1–3 ms)**. Also primes the owner cache for the hits' follow-up requests. + +## 2. Batch-ZIP downloads — no per-file authz/Recent (ZIP-BATCH-AUTHZ) + +`BatchOperations::add_folder_subtree_to_zip` had already authorized the +subtree ROOT (`get_folder_with_perms`), yet every enumerated file still paid +`get_file_stream_with_perms` = 1 authz point SELECT + a Recent-hook spawn +issuing 2 writes (INSERT … ON CONFLICT + prune DELETE). A 2,000-file folder +ZIP ⇒ ~6,000 extra statements. Subtree entries now use the plain +`get_file_stream` — exactly what `ZipService::create_folder_zip` (the native +folder-download path) has always done. Explicitly-selected top-level files +keep per-file authz + Recent. Unit price: DEAD-PROPS.md sequential rows — +**~1.5–4.5 s of DB chatter removed** from a 2,000-file archive, plus the ZIP +no longer floods Recents with every archived file. + +## 3. CDC manifest RAM cache (MANIFEST-CACHE) + +Every stream / range / full read of a CDC blob paid one +`chunk_manifests` row fetch first — p50 0.44 ms (4.4 ms under pool pressure, +benches/BLOB-MANIFEST.md), on the hottest read paths there are (media +serving, thumbnails, range seeks). Manifests are immutable by content +address, so `DedupService` now memoises them (moka, weight-bounded 32 MiB, +60 s TTL, positive-only so background rechunking is honoured immediately; +invalidated post-commit on the two delete paths). Warm read: **0.44–4.4 ms → +~1 µs** (CHROOT-CACHE.md's moka row) and one fewer pool checkout per read — +range-seek storms (video scrubbing) hit this every request. + +## 4. Public share landing — 3 round-trips → 1 atomic UPDATE (SHARE-ACCESS) + +`GET /api/s/{token}` ran find_share_by_token (with a correlated +`MIN(expires_at)` subquery), a full-row UPDATE writing back a Rust-side +increment (racy: lost updates between concurrent visitors, and it rewrote +`item_name`/`password_hash` wholesale — clobbering concurrent owner edits), +then the handler's follow-up fetched the share a third time. +`ShareStoragePort::increment_access_count` is now one +`UPDATE … SET access_count = access_count + 1 WHERE token = $1 AND `: +**3 subquery round-trips → 2** for the landing (register + fetch), no +read-modify-write race, no collateral column rewrites. + +## 5. Trash — dead SELECT removed + +`TrashService::move_to_trash` fetched the full file/folder entity to build a +`TrashedItem` consumed only by `TrashRepository::add_to_trash` — a documented +no-op in the soft-delete model. Both branches now go straight to the +`move_to_trash` UPDATE: **one uncached SELECT + entity hydration removed per +trash operation** (file and folder). + +## 6. NFC normalization fast path + +`normalize_storage_name` ran unicode-normalization's full +decompose/recompose state machine on every name of every row loaded from PG +(listings, PROPFIND, photos — 27 constructor call sites), even though the DB +invariant guarantees stored names are already NFC. `is_nfc_quick` (a +per-char table lookup) now short-circuits the ~100 % case to a plain copy; +`Maybe`/`No` still run the full pipeline, so semantics are unchanged. + +## 7. Frontend — first-page render for large folders + +`fetchFolderListing` paged the ENTIRE folder (sequential 200-item requests) +before returning anything — a 2,000-item folder waited ~10 round-trips +before first paint. The files route now paints page one immediately via the +new `onPage` hook and fills in as later pages land (skipped when a cached +listing is already on screen, so views never shrink). First-paint latency +for an N-item folder drops from ⌈N/200⌉ sequential RTTs to 1. + +## Refuted by benchmark (reverted, kept for the record) + +- **Cached `Intl.Collator` for name sorts (frontend):** sorting 5,000 names — + argument-less `localeCompare` 5.6 ms vs cached collator **12.1 ms (2× + slower)**. V8 fast-paths argument-less `localeCompare`; the "cache the + collator" folklore does not apply. Reverted, ordering untouched. diff --git a/benches/PEOPLE-LIST.md b/benches/PEOPLE-LIST.md new file mode 100644 index 00000000..06c03f41 --- /dev/null +++ b/benches/PEOPLE-LIST.md @@ -0,0 +1,35 @@ +# People tab — grouped COUNT (vs full faces scan with embeddings) + +`PeopleService::list_people` (GET `/api/people`, fetched on every People-tab +mount) called `faces_for_user`, which SELECTs every face row for the caller — +each carrying a 2,048-byte embedding BYTEA that gets decoded into a fresh +`Vec` — only to (a) count faces per person and (b) resolve a handful of +cover faces to file ids. A 10k-face library moved ~21 MB of embeddings per +request. `merge()` had the same over-fetch plus one UPDATE per face. + +Changes (`FaceRepository` + `PeopleService`): + +- `person_face_stats`: `SELECT person_id, COUNT(*) … GROUP BY person_id`. +- `file_ids_for_faces`: one `id = ANY($1)` over just the cover face ids. +- `reassign_person_faces`: merge as ONE set-based UPDATE (was: load all + faces, filter in Rust, one UPDATE per face). + +## Reproduce + +```bash +cargo run --release --features bench --example bench_people_list +# tunables: BENCH_FACES=10000 BENCH_PERSONS=20 BENCH_REPS=5 +``` + +## Results (4 cores, local PG16, 10,000 faces / 20 persons) + +| mode | total ms | bytes moved | +|--------------------------|---------:|------------:| +| BEFORE — full face rows | 30.40 | 20,960,000 | +| AFTER — COUNT + covers | 3.76 | 1,280 | + +- **8.1× faster** and **~16,000× fewer bytes** off the wire per People-tab + mount. The heap never materialises 10k embedding `Vec`s. +- The BEFORE row also allocated ~21 MB per request on the server; under a + handful of concurrent mounts that was tens of MB of transient RSS for a + page that shows 20 avatars. diff --git a/benches/PROPFIND-PAGING.md b/benches/PROPFIND-PAGING.md new file mode 100644 index 00000000..b4b2eee3 --- /dev/null +++ b/benches/PROPFIND-PAGING.md @@ -0,0 +1,51 @@ +# PROPFIND folder paging — keyset cursor + (folder_id, name) index + +`list_files_batch` walks a folder's children in name order, 500 per page +(native + NextCloud PROPFIND streamers). The old shape was `ORDER BY name +LIMIT 500 OFFSET k` with **no supporting index** — the initial schema's +`(folder_id, name, user_id)` index that served it was dropped by migration +20260902000000 (user_id → nullable), leaving only `idx_files_folder_id`. So +every page bitmap-scanned all N children and top-sorted them: a full listing +of an N-file folder cost O(N²/500) row visits + ⌈N/500⌉ sorts. + +Changes: + +1. Migration `20260917000000_files_folder_name_index.sql`: partial composite + `idx_files_folder_name (folder_id, name) WHERE NOT is_trashed`. +2. `list_files_batch` cursor switched from OFFSET to keyset + (`name > $last`, names are unique per folder via the + `(drive_id, folder_id, name)` unique index) across the port trait, the + repository and both handler loops. The cursor predicate is only emitted + when a cursor exists — a `$2 IS NULL OR …` disjunction would block the + index condition under the extended protocol's generic plans. + +## Reproduce + +```bash +cargo run --release --features bench --example bench_propfind_paging +# tunables: BENCH_FILES=20000 BENCH_PAGE=500 BENCH_REPS=3 +``` + +Times the FULL page-by-page walk of a 20,000-file folder (the listing +portion of one Depth:1 PROPFIND). + +## Results (4 cores, local PG16) + +| mode | total ms | vs OLD | +|----------------------------------|---------:|-------:| +| OFFSET, no index (true BEFORE) | 1,266.3 | 1.0× | +| OFFSET + index (index alone) | 482.7 | 2.6× | +| KEYSET + index (AFTER) | 76.7 | **16.5×** | + +- Full-folder listing cost drops **16.5×**; unlike OFFSET (even indexed), + keyset stays O(page) at any depth, so the gap widens with folder size. +- Companion fix in the same commit: the Photos timeline cursor + (`list_media_files`) wrapped its keyset column in + `EXTRACT(EPOCH FROM …)::bigint` plus an `IS NULL OR` disjunction — + non-sargable, so page k re-scanned all k·limit rows already scrolled past. + It now compares the raw `media_sort_date` against a timestamptz bind + (identical row semantics — the cursor is whole seconds) and splits the + cursor/no-cursor query shapes, restoring the + `idx_files_media_timeline_by_drive` boundary condition the index was built + for. Same mechanism as measured above (index-boundary vs per-row filter); + the deep-scroll effect mirrors the OFFSET column. diff --git a/benches/QUOTA-PATH.md b/benches/QUOTA-PATH.md new file mode 100644 index 00000000..7de86418 --- /dev/null +++ b/benches/QUOTA-PATH.md @@ -0,0 +1,42 @@ +# Quota path — narrow 2-column read + skip-when-not-requested + +Two independent fixes on the quota resolution that runs on every upload check +and every quota-reporting folder PROPFIND: + +1. **Narrow read.** `check_storage_quota` / `get_user_storage_info` called + `get_user_by_id`, whose SELECT drags the entire `auth.users` row — + including `image`, an avatar data URI of up to 512 KiB — to read two i64s. + New `UserPgRepository::get_storage_usage` reads exactly + `(storage_used_bytes, storage_quota_bytes)` (same pattern as the existing + `get_user_flags`). +2. **Skip entirely when not asked.** `resolve_webdav_quota` (2 round-trips: + drive row + user row) ran on EVERY folder PROPFIND on both surfaces, even + when the client's `` list named no quota property — which is the + common shape for sync-client polls. `PropFindRequest::wants_quota()` now + gates it: `AllProp`/`PropName` keep quota (the writers emit RFC 4331 props + there), explicit prop lists trigger the lookups only if they name + `quota-used-bytes` / `quota-available-bytes`. Responses are byte-identical + for every request that names quota or asks for allprop. + +## Reproduce + +```bash +cargo run --release --features bench --example bench_quota_path +# tunables: BENCH_SECONDS=4 BENCH_CONCURRENCIES=8,64 BENCH_IMAGE_KB=512 +``` + +## Results (4 cores, local PG16, pool=20, 512 KiB avatar on the row) + +| conc | mode | ops/s | p50 µs | p99 µs | +|-----:|--------|-------:|---------:|---------:| +| 8 | FULL | 2,222 | 3,369.4 | 8,452.2 | +| 8 | NARROW | 25,118 | 294.9 | 867.9 | +| 64 | FULL | 2,567 | 24,642.3 | 36,164.0 | +| 64 | NARROW | 40,195 | 1,468.9 | 3,964.9 | + +- **11–16× throughput, p50 3.4 ms → 0.29 ms** for the user-row half of every + quota resolution (the avatar bytes dominated the wire+decode cost). +- With `wants_quota()` the common PROPFIND pays **zero** quota queries — the + numbers above then only apply to requests that actually ask for quota. +- The same narrow read protects every upload (`check_storage_quota` gates all + upload paths), where the FULL row was pure overhead per file. diff --git a/benches/STATIC-PRECOMPRESSED.md b/benches/STATIC-PRECOMPRESSED.md new file mode 100644 index 00000000..f6630433 --- /dev/null +++ b/benches/STATIC-PRECOMPRESSED.md @@ -0,0 +1,56 @@ +# Static assets & API responses — precompressed siblings + explicit Brotli level + +Two related findings, one root cause: tower-http's `CompressionLayer` default +maps to **Brotli QUALITY 11** (`async-compression Level::Default` → +`BrotliEncoderParams::default()`, brotli-8.0.2 `encode.rs:323` — verified in +source and empirically below). Quality 11 is a deploy-time setting; it was +running per request on: + +- every SPA asset (`interfaces/web/mod.rs` layer): ~1.3 s CPU per 700 KiB + bundle per request; +- every compressible API response (`main.rs` global layer): ~90 ms CPU per + 64 KiB JSON response. + +Changes: + +1. **Precompressed statics.** `frontend/scripts/precompress.mjs` (build step, + node:zlib only) emits `.br`/`.gz` siblings for text assets; `ServeDir` now + uses `precompressed_br()/precompressed_gzip()` — a request costs a file + read, and clients get the *better* q11 bytes, paid once per deploy + (~1.4 s for the whole bundle). +2. **Explicit level 4** on both `CompressionLayer`s + (`CompressionLevel::Precise(4)`) — the on-the-fly fallback for statics + without siblings, and the global API layer. + +## Reproduce + +```bash +cargo run --release --features bench --example bench_static_precompress +# tunables: BENCH_ASSET_KB=700 BENCH_REPS=30 +``` + +## Results (4 cores, this container) + +**Per-request cost, 700 KiB JS-like asset (94 % compressible):** + +| mode | ms/request | speedup | +|-----------------------------|-----------:|--------:| +| BEFORE — on-the-fly Brotli | 1,324.31 | 1.0× | +| AFTER — precompressed read | 0.657 | **2016×** | + +**Brotli level sweep, 64 KiB JSON-like API response:** + +| level | ms/resp | out KiB | +|-------------------------|--------:|--------:| +| Default (= quality 11!) | 90.10 | 5.4 | +| **Precise(4)** (chosen) | 0.91 | 6.2 | +| Fastest | 0.15 | 9.3 | + +- Statics: 3 orders of magnitude less CPU per request, while shipping + *smaller* bytes than the runtime default would at any reasonable level. +- API responses: **99× less CPU** for ~15 % more bytes (5.4 → 6.2 KiB) — + `Precise(4)` is the classic dynamic-content operating point; `Fastest` + gives up too much density (9.3 KiB). +- Historical note: an earlier review round REFUTED the "default is q11" + claim twice; the source line and the 90 ms/64 KiB measurement above settle + it the other way. Measure before believing — in both directions. diff --git a/benches/ZIP-MEDIA.md b/benches/ZIP-MEDIA.md new file mode 100644 index 00000000..41d61b17 --- /dev/null +++ b/benches/ZIP-MEDIA.md @@ -0,0 +1,44 @@ +# ZIP export — Stored for already-compressed media (vs Deflate-always) + +Every ZIP export path (`ZipService::create_folder_zip` for folder downloads + +public share ZIPs, `BatchOperations::download_zip` for batch downloads) used to +build **every** file entry with `Compression::Deflate`. The dominant "download +folder" payload is photos/video (JPEG/HEIC/MP4/WebP), which deflate cannot +shrink (~0 %) while costing ~40 MB/s of CPU per core — and `async_zip` runs +deflate **inline on the writing tokio task** (inside `poll_write`), so a media +folder download monopolised ~1 core for its whole duration. + +The change picks the entry compression from the file's MIME type at plan time: +`Stored` for already-compressed content, `Deflate` otherwise. The shared +predicate is `common::mime_detect::is_precompressed_mime` / +`zip_entry_compression` — it mirrors the HTTP `CompressionLayer` exclusion +list in `main.rs` (keep in sync), minus `x-tar`/`octet-stream` (containers of +possibly-compressible data stay on Deflate so nothing ever gets bigger). + +## Reproduce + +```bash +cargo run --release --features bench --example bench_zip_media +# tunables: BENCH_MEDIA_FILES=48 BENCH_MEDIA_MB=4 BENCH_TEXT_FILES=24 BENCH_TEXT_MB=2 BENCH_REPS=3 +``` + +Rebuilds the exact production writer stack (`ZipFileWriter::with_tokio(BufWriter(File))`, +`write_entry_stream`, 64 KiB chunks) over a mixed corpus: 192 MiB incompressible +"media" + 48 MiB compressible text (80/20 by bytes, a realistic media folder). + +## Results (4 cores, this container) + +| mode | wall s | cpu s | MB/s | out MiB | speedup | +|-----------------------|-------:|------:|-------:|--------:|--------:| +| all-Deflate (BEFORE) | 5.786 | 5.88 | 41.5 | 198.8 | 1.00× | +| mime-aware (AFTER) | 1.341 | 1.38 | 178.9 | 198.7 | **4.31×** | +| all-Stored (bound) | 0.150 | 0.19 | 1601.5 | 240.0 | 38.6× | + +- **4.31× faster wall clock and 4.3× less CPU** on the mixed corpus, with the + archive **0.05 % smaller** (media never deflated anyway; text keeps Deflate). +- The remaining 1.38 s CPU in mime-aware is the text deflate + CRC32 — the + irreducible part. Pure-media folders approach the all-Stored bound (the + archive becomes blob-read-bound instead of CPU-bound). +- Side effect on the runtime: the writing task no longer occupies ~a full core + per media download — on a 4-core box that's ~25 % of total CPU handed back + to other requests for the duration of every archive. diff --git a/examples/bench_chroot_cache.rs b/examples/bench_chroot_cache.rs new file mode 100644 index 00000000..94dd113a --- /dev/null +++ b/examples/bench_chroot_cache.rs @@ -0,0 +1,250 @@ +//! NC chroot / default-drive resolution benchmark — 2 queries/request vs moka. +//! +//! The NextCloud basic-auth middleware wraps EVERY protected NC route and, +//! even with app-password verification fully cached, used to resolve the +//! chroot from scratch per request: +//! +//! 1. `find_default_for_user` — drives JOIN folders (drive_pg_repository) +//! 2. `get_folder(root_id)` — folders by PK +//! +//! The native `/webdav` surface repeats query 1 per request (Mode-B scope +//! resolution), WOPI repeats it per call. The change memoises (1) inside +//! `DrivePgRepository` and (2) in the middleware's `NC_CHROOT_CACHE` +//! (both 30 s TTL). This bench isolates exactly that: the per-request DB +//! cost of the chroot resolution — the two production query shapes vs a +//! moka hit — under sync-storm concurrency against the real pool. +//! +//! Run (needs Postgres up; reads DATABASE_URL from .env): +//! cargo run --release --features bench --example bench_chroot_cache +//! Tunables (env): BENCH_POOL (20), BENCH_SECONDS (4), BENCH_CONCURRENCIES ("8,64"). + +use std::env; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use sqlx::postgres::PgPoolOptions; +use sqlx::{PgPool, Row}; +use uuid::Uuid; + +fn env_or(key: &str, default: T) -> T { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +struct Seeded { + user_id: Uuid, +} + +async fn seed(pool: &PgPool) -> Seeded { + // user → (drive + root folder + root_folder_id stamp) in one tx — + // trg_no_orphan_root_folder is INITIALLY DEFERRED and checks at commit. + let mut tx = pool.begin().await.expect("begin"); + let user_id: Uuid = sqlx::query_scalar( + "INSERT INTO auth.users (username, email, role) + VALUES ('bench_chroot', 'bench_chroot@bench.invalid', 'user') + RETURNING id", + ) + .fetch_one(&mut *tx) + .await + .expect("seed user"); + let drive_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.drives (kind, default_for_user) VALUES ('personal', $1) RETURNING id", + ) + .bind(user_id) + .fetch_one(&mut *tx) + .await + .expect("seed drive"); + let folder_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.folders (name, path, lpath, drive_id) + VALUES ('Personal', '/Personal', 'Personal', $1) RETURNING id", + ) + .bind(drive_id) + .fetch_one(&mut *tx) + .await + .expect("seed folder"); + sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2") + .bind(folder_id) + .bind(drive_id) + .execute(&mut *tx) + .await + .expect("stamp root"); + tx.commit().await.expect("commit"); + Seeded { user_id } +} + +async fn cleanup(pool: &PgPool, user_id: Uuid) { + let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1") + .bind(user_id) + .execute(pool) + .await; +} + +/// The exact production BEFORE: both chroot queries, sequentially (the +/// middleware awaits the drive row to learn root_folder_id first). +async fn one_op_before(pool: &PgPool, user_id: Uuid, queries: &AtomicUsize) { + let row = sqlx::query( + r#" + SELECT d.id, d.kind, d.default_for_user, d.root_folder_id, + d.quota_bytes, d.used_bytes, d.policies, + d.created_at, d.updated_at, + f.name AS root_folder_name + FROM storage.drives d + JOIN storage.folders f ON f.id = d.root_folder_id + WHERE d.default_for_user = $1 + "#, + ) + .bind(user_id) + .fetch_one(pool) + .await + .expect("drive query"); + let root_id: Uuid = row.get("root_folder_id"); + + let _folder = sqlx::query( + "SELECT id, name, parent_id, path, created_at, updated_at + FROM storage.folders WHERE id = $1", + ) + .bind(root_id) + .fetch_one(pool) + .await + .expect("folder query"); + queries.fetch_add(2, Ordering::Relaxed); +} + +#[derive(Clone)] +#[allow(dead_code)] +struct ChrootValue { + root_id: Uuid, + name: String, + path: String, +} + +struct Stats { + rps: f64, + p50: f64, + p95: f64, + p99: f64, +} + +fn summarize(mut lats: Vec, secs: u64) -> Stats { + lats.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let n = lats.len(); + let pct = |p: f64| { + if n == 0 { + 0.0 + } else { + lats[((n as f64 * p) as usize).min(n - 1)] + } + }; + Stats { + rps: n as f64 / secs as f64, + p50: pct(0.50), + p95: pct(0.95), + p99: pct(0.99), + } +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + dotenvy::dotenv().ok(); + let url = env::var("DATABASE_URL") + .or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING")) + .expect("set DATABASE_URL — the dev Postgres URL"); + + let pool_size: u32 = env_or("BENCH_POOL", 20); + let secs: u64 = env_or("BENCH_SECONDS", 4); + let concurrencies: Vec = env::var("BENCH_CONCURRENCIES") + .ok() + .map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect()) + .unwrap_or_else(|| vec![8, 64]); + + let pool = Arc::new( + PgPoolOptions::new() + .max_connections(pool_size) + .min_connections(pool_size) + .acquire_timeout(Duration::from_secs(10)) + .connect(&url) + .await + .expect("connect Postgres"), + ); + + let seeded = seed(&pool).await; + let user_id = seeded.user_id; + + // AFTER: what the middleware pays on a warm cache — a moka lookup. + let cache: moka::sync::Cache = moka::sync::Cache::builder() + .max_capacity(100_000) + .time_to_live(Duration::from_secs(30)) + .build(); + cache.insert( + user_id, + ChrootValue { + root_id: Uuid::new_v4(), + name: "Personal".into(), + path: "/Personal".into(), + }, + ); + + println!("\n#############################################################"); + println!("# NC chroot resolution: BEFORE (2 queries/req) vs AFTER (moka)"); + println!("# pool={pool_size} window={secs}s/run"); + println!("#############################################################\n"); + println!( + "| {:>5} | {:<6} | {:>10} | {:>9} | {:>9} | {:>9} | {:>9} |", + "conc", "mode", "req/s", "p50 µs", "p95 µs", "p99 µs", "queries" + ); + + for &conc in &concurrencies { + for mode in ["BEFORE", "AFTER"] { + let queries = Arc::new(AtomicUsize::new(0)); + let deadline = Instant::now() + Duration::from_secs(secs); + let mut handles = Vec::new(); + for _ in 0..conc { + let pool = pool.clone(); + let cache = cache.clone(); + let queries = queries.clone(); + let mode = mode.to_string(); + handles.push(tokio::spawn(async move { + let mut lats = Vec::new(); + while Instant::now() < deadline { + let t = Instant::now(); + if mode == "BEFORE" { + one_op_before(&pool, user_id, &queries).await; + } else { + let v = cache.get(&user_id).expect("warm cache"); + std::hint::black_box(v); + } + lats.push(t.elapsed().as_secs_f64() * 1_000_000.0); + if mode == "AFTER" { + // moka hit is ~100 ns; yield so the loop doesn't + // monopolise workers and skew the run count. + tokio::task::yield_now().await; + } + } + lats + })); + } + let mut all = Vec::new(); + for h in handles { + all.extend(h.await.unwrap()); + } + let s = summarize(all, secs); + println!( + "| {:>5} | {:<6} | {:>10.0} | {:>9.2} | {:>9.2} | {:>9.2} | {:>9} |", + conc, + mode, + s.rps, + s.p50, + s.p95, + s.p99, + queries.load(Ordering::Relaxed) + ); + } + } + + cleanup(&pool, user_id).await; + println!("\n(BEFORE = the two production chroot queries; AFTER = warm moka hit."); + println!(" Every NC request pays this before its handler runs.)"); +} diff --git a/examples/bench_dead_props.rs b/examples/bench_dead_props.rs new file mode 100644 index 00000000..5555404a --- /dev/null +++ b/examples/bench_dead_props.rs @@ -0,0 +1,286 @@ +//! WebDAV dead-properties fetch benchmark — per-child N+1 vs batched ANY($1). +//! +//! The streaming PROPFIND walker (`webdav_handler.rs`) fetches dead properties +//! ONE CHILD AT A TIME, sequentially, for every Depth:1 listing page: +//! +//! for file in &batch { file_deads.push(store.get_all(File(id)).await) } +//! +//! and `DeadPropertyStore::get_all` filters with +//! `folder_id IS NOT DISTINCT FROM $1 AND file_id IS NOT DISTINCT FROM $2`, +//! which PostgreSQL cannot serve from a B-tree index (IS NOT DISTINCT FROM is +//! not an indexable operator) — so each of the N sequential round-trips also +//! degrades to a seq scan as the table grows. +//! +//! This bench isolates exactly the dead-prop portion of a Depth:1 PROPFIND of +//! a folder with N children, comparing the three query shapes: +//! +//! OLD — N sequential `IS NOT DISTINCT FROM` queries (production today) +//! EQ — N sequential plain `file_id = $1` queries (indexable, still N+1) +//! BATCH — ⌈N/500⌉ `file_id = ANY($1)` queries (one per PROPFIND page) +//! +//! Two table sizes are measured: the seeded-children-only table and one with +//! extra noise rows (dead props on other resources), which is where the +//! seq-scan cost of OLD shows up. +//! +//! Run (needs Postgres up; reads DATABASE_URL from .env): +//! cargo run --release --features bench --example bench_dead_props +//! Tunables (env): BENCH_CHILDREN (2000), BENCH_PAGE (500 = PROPFIND_BATCH_SIZE), +//! BENCH_NOISE_ROWS (20000), BENCH_REPS (5). + +use std::env; +use std::time::Instant; + +use sqlx::postgres::PgPoolOptions; +use sqlx::{PgPool, Row}; +use uuid::Uuid; + +fn env_or(key: &str, default: T) -> T { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +struct Seeded { + drive_id: Uuid, + file_ids: Vec, +} + +async fn seed(pool: &PgPool, children: usize, noise: usize) -> Seeded { + // Drive (kind 'shared' needs no user FK) → root folder → N files → props. + // The root folder + drive.root_folder_id must land in ONE transaction: + // trg_no_orphan_root_folder is INITIALLY DEFERRED and checks at commit. + let mut tx = pool.begin().await.expect("begin seed tx"); + let drive_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id", + ) + .fetch_one(&mut *tx) + .await + .expect("seed drive"); + + let folder_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.folders (name, path, lpath, drive_id) + VALUES ('bench_dead_props', '/bench_dead_props', 'bench_dead_props', $1) + RETURNING id", + ) + .bind(drive_id) + .fetch_one(&mut *tx) + .await + .expect("seed folder"); + + sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2") + .bind(folder_id) + .bind(drive_id) + .execute(&mut *tx) + .await + .expect("stamp root_folder_id"); + tx.commit().await.expect("commit seed tx"); + + // Children of the PROPFIND'd folder, one dead prop each. + let file_ids: Vec = sqlx::query_scalar( + "INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id) + SELECT 'f' || i, $1, 'benchdead000000000000000000000000000000000000000000000000000000', + 1024, 'image/jpeg', $2 + FROM generate_series(1, $3) AS i + RETURNING id", + ) + .bind(folder_id) + .bind(drive_id) + .bind(children as i32) + .fetch_all(pool) + .await + .expect("seed files"); + + sqlx::query( + "INSERT INTO storage.webdav_dead_properties (file_id, namespace, local_name, value) + SELECT id, 'urn:bench', 'displayname', 'bench value' + FROM storage.files WHERE folder_id = $1", + ) + .bind(folder_id) + .execute(pool) + .await + .expect("seed dead props"); + + // Noise: dead props attached to OTHER files (a second folder) so the + // table has realistic volume — this is what OLD's seq scans pay for. + if noise > 0 { + // Child of the main folder — root folders need the deferred + // four-write dance, children don't. + let noise_folder: Uuid = sqlx::query_scalar( + "INSERT INTO storage.folders (name, parent_id, path, lpath, drive_id) + VALUES ('noise', $2, '/bench_dead_props/noise', 'bench_dead_props.noise', $1) + RETURNING id", + ) + .bind(drive_id) + .bind(folder_id) + .fetch_one(pool) + .await + .expect("seed noise folder"); + sqlx::query( + "WITH f AS ( + INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id) + SELECT 'n' || i, $1, 'benchdead000000000000000000000000000000000000000000000000000000', + 1024, 'image/jpeg', $2 + FROM generate_series(1, $3) AS i + RETURNING id + ) + INSERT INTO storage.webdav_dead_properties (file_id, namespace, local_name, value) + SELECT id, 'urn:bench', 'noise', 'x' FROM f", + ) + .bind(noise_folder) + .bind(drive_id) + .bind(noise as i32) + .execute(pool) + .await + .expect("seed noise props"); + } + + sqlx::query("ANALYZE storage.webdav_dead_properties") + .execute(pool) + .await + .ok(); + + Seeded { drive_id, file_ids } +} + +async fn cleanup(pool: &PgPool, drive_id: Uuid) { + // drives → folders/files → dead props all cascade. + let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1") + .bind(drive_id) + .execute(pool) + .await; +} + +/// OLD: production `get_all` shape — sequential, IS NOT DISTINCT FROM. +async fn run_old(pool: &PgPool, ids: &[Uuid]) -> usize { + let mut rows_seen = 0; + for id in ids { + let rows = sqlx::query( + "SELECT namespace, local_name, value + FROM storage.webdav_dead_properties + WHERE folder_id IS NOT DISTINCT FROM $1 + AND file_id IS NOT DISTINCT FROM $2", + ) + .bind(Option::::None) + .bind(Some(*id)) + .fetch_all(pool) + .await + .expect("old get_all"); + rows_seen += rows.len(); + } + rows_seen +} + +/// EQ: still N sequential round-trips, but with an indexable `=` predicate. +async fn run_eq(pool: &PgPool, ids: &[Uuid]) -> usize { + let mut rows_seen = 0; + for id in ids { + let rows = sqlx::query( + "SELECT namespace, local_name, value + FROM storage.webdav_dead_properties + WHERE file_id = $1", + ) + .bind(*id) + .fetch_all(pool) + .await + .expect("eq get_all"); + rows_seen += rows.len(); + } + rows_seen +} + +/// BATCH: one `= ANY($1)` query per PROPFIND page of 500 children. +async fn run_batch(pool: &PgPool, ids: &[Uuid], page: usize) -> usize { + let mut rows_seen = 0; + for chunk in ids.chunks(page) { + let rows = sqlx::query( + "SELECT file_id, namespace, local_name, value + FROM storage.webdav_dead_properties + WHERE file_id = ANY($1)", + ) + .bind(chunk) + .fetch_all(pool) + .await + .expect("batch get_all"); + // Decode file_id like the real batched store method will (map key). + for row in &rows { + let _: Uuid = row.get("file_id"); + } + rows_seen += rows.len(); + } + rows_seen +} + +fn median(mut xs: Vec) -> f64 { + xs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + xs[xs.len() / 2] +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + dotenvy::dotenv().ok(); + let url = env::var("DATABASE_URL") + .or_else(|_| env::var("OXICLOUD_DB_CONNECTION_STRING")) + .expect("set DATABASE_URL — the dev Postgres URL"); + + let children: usize = env_or("BENCH_CHILDREN", 2000); + let page: usize = env_or("BENCH_PAGE", 500); + let noise: usize = env_or("BENCH_NOISE_ROWS", 20_000); + let reps: usize = env_or("BENCH_REPS", 5); + + let pool = PgPoolOptions::new() + .max_connections(5) + .min_connections(5) + .connect(&url) + .await + .expect("connect Postgres"); + + for &with_noise in &[false, true] { + let n = if with_noise { noise } else { 0 }; + let seeded = seed(&pool, children, n).await; + let total_rows: i64 = + sqlx::query_scalar("SELECT count(*) FROM storage.webdav_dead_properties") + .fetch_one(&pool) + .await + .unwrap_or(0); + + println!("\n== folder with {children} children, dead-props table = {total_rows} rows =="); + println!( + "{:<28} {:>10} {:>12} {:>9}", + "mode", "queries", "total ms", "vs OLD" + ); + + let mut base = None; + for (label, queries) in [ + ("OLD seq, IS NOT DISTINCT", children), + ("EQ seq, file_id = $1", children), + ("BATCH file_id = ANY, /page", children.div_ceil(page)), + ] { + let mut times = Vec::with_capacity(reps); + let mut rows = 0; + for _ in 0..reps { + let t = Instant::now(); + rows = match label.split_whitespace().next().unwrap() { + "OLD" => run_old(&pool, &seeded.file_ids).await, + "EQ" => run_eq(&pool, &seeded.file_ids).await, + _ => run_batch(&pool, &seeded.file_ids, page).await, + }; + times.push(t.elapsed().as_secs_f64() * 1000.0); + } + assert_eq!(rows, children, "each child has exactly 1 dead prop"); + let ms = median(times); + let speedup = base + .map(|b: f64| format!("{:.1}x", b / ms)) + .unwrap_or_else(|| "1.0x".into()); + if base.is_none() { + base = Some(ms); + } + println!("{label:<28} {queries:>10} {ms:>12.2} {speedup:>9}"); + } + + cleanup(&pool, seeded.drive_id).await; + } + + println!("\n(total ms = the dead-prop portion of one Depth:1 PROPFIND of the folder,"); + println!(" i.e. what the walker adds on top of the file/folder listing queries)"); +} diff --git a/examples/bench_people_list.rs b/examples/bench_people_list.rs new file mode 100644 index 00000000..0dd4c1d6 --- /dev/null +++ b/examples/bench_people_list.rs @@ -0,0 +1,213 @@ +//! People-tab benchmark — full faces scan (embeddings included) vs grouped COUNT. +//! +//! `PeopleService::list_people` used to call `faces_for_user`, dragging every +//! face row — each with a 2,048-byte embedding BYTEA — across the wire and +//! decoding it into a fresh `Vec`, only to (a) count faces per person and +//! (b) resolve ~a-handful of cover faces to file ids. The change replaces it +//! with `person_face_stats` (grouped COUNT) + `file_ids_for_faces` (one +//! `= ANY` over just the cover ids). +//! +//! Run (needs Postgres up; reads DATABASE_URL from .env): +//! cargo run --release --features bench --example bench_people_list +//! Tunables: BENCH_FACES (10000), BENCH_PERSONS (20), BENCH_REPS (5) + +use std::env; +use std::time::Instant; + +use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; +use uuid::Uuid; + +fn env_or(key: &str, default: T) -> T { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +struct Seeded { + user_id: Uuid, + drive_id: Uuid, + cover_ids: Vec, +} + +async fn seed(pool: &PgPool, faces: usize, persons: usize) -> Seeded { + let mut tx = pool.begin().await.expect("begin"); + let user_id: Uuid = sqlx::query_scalar( + "INSERT INTO auth.users (username, email, role) + VALUES ('bench_people', 'bench_people@bench.invalid', 'user') RETURNING id", + ) + .fetch_one(&mut *tx) + .await + .expect("user"); + let drive_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.drives (kind, default_for_user) VALUES ('personal', $1) RETURNING id", + ) + .bind(user_id) + .fetch_one(&mut *tx) + .await + .expect("drive"); + let folder_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.folders (name, path, lpath, drive_id) + VALUES ('bench_people', '/bench_people', 'bench_people', $1) RETURNING id", + ) + .bind(drive_id) + .fetch_one(&mut *tx) + .await + .expect("folder"); + sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2") + .bind(folder_id) + .bind(drive_id) + .execute(&mut *tx) + .await + .expect("stamp root"); + tx.commit().await.expect("commit"); + + // Photo files the faces point at. + let file_ids: Vec = sqlx::query_scalar( + "INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id) + SELECT 'p' || i, $1, 'benchpeople0000000000000000000000000000000000000000000000000000', + 1024, 'image/jpeg', $2 + FROM generate_series(1, $3) AS i + RETURNING id", + ) + .bind(folder_id) + .bind(drive_id) + .bind(faces as i32) + .fetch_all(pool) + .await + .expect("files"); + + // Persons + faces (2 KiB embedding each, like the real 512×f32). + let mut person_ids = Vec::with_capacity(persons); + for i in 0..persons { + let pid: Uuid = sqlx::query_scalar( + "INSERT INTO faces.persons (user_id, display_name) VALUES ($1, $2) RETURNING id", + ) + .bind(user_id) + .bind(format!("Person {i}")) + .fetch_one(pool) + .await + .expect("person"); + person_ids.push(pid); + } + + let embedding = vec![0u8; 2048]; + let mut cover_ids = Vec::with_capacity(persons); + for (i, file_id) in file_ids.iter().enumerate() { + let pid = person_ids[i % persons]; + let face_id: Uuid = sqlx::query_scalar( + "INSERT INTO faces.faces + (file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash) + VALUES ($1, $2, $3, ARRAY[0.1,0.1,0.2,0.2]::real[], 0.99, 0.9, $4, + 'benchpeople0000000000000000000000000000000000000000000000000000') + RETURNING id", + ) + .bind(file_id) + .bind(user_id) + .bind(pid) + .bind(&embedding) + .fetch_one(pool) + .await + .expect("face"); + if i < persons { + cover_ids.push(face_id); + } + } + sqlx::query("ANALYZE faces.faces").execute(pool).await.ok(); + + Seeded { + user_id, + drive_id, + cover_ids, + } +} + +async fn cleanup(pool: &PgPool, s: &Seeded) { + let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1") + .bind(s.drive_id) + .execute(pool) + .await; + let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1") + .bind(s.user_id) + .execute(pool) + .await; +} + +fn median(mut xs: Vec) -> f64 { + xs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + xs[xs.len() / 2] +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + dotenvy::dotenv().ok(); + let url = env::var("DATABASE_URL").expect("set DATABASE_URL"); + let faces: usize = env_or("BENCH_FACES", 10_000); + let persons: usize = env_or("BENCH_PERSONS", 20); + let reps: usize = env_or("BENCH_REPS", 5); + + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(&url) + .await + .expect("connect"); + println!("seeding {faces} faces / {persons} persons (one-time)…"); + let seeded = seed(&pool, faces, persons).await; + + println!( + "\n# GET /api/people data fetch: BEFORE (full face rows) vs AFTER (COUNT + cover ANY)" + ); + println!("{:<28} {:>12} {:>14}", "mode", "total ms", "bytes moved"); + + let mut base = None; + for mode in ["BEFORE full-rows", "AFTER count+covers"] { + let mut times = Vec::with_capacity(reps); + let mut bytes = 0usize; + for _ in 0..reps { + let t = Instant::now(); + if mode.starts_with("BEFORE") { + // faces_for_user shape: every column incl. embedding. + let rows: Vec<(Uuid, Uuid, Option, Vec)> = sqlx::query_as( + "SELECT id, file_id, person_id, embedding FROM faces.faces WHERE user_id = $1", + ) + .bind(seeded.user_id) + .fetch_all(&pool) + .await + .expect("full rows"); + bytes = rows.iter().map(|r| r.3.len() + 48).sum(); + assert_eq!(rows.len(), faces); + } else { + let stats: Vec<(Uuid, i64)> = sqlx::query_as( + "SELECT person_id, COUNT(*) FROM faces.faces + WHERE user_id = $1 AND person_id IS NOT NULL GROUP BY person_id", + ) + .bind(seeded.user_id) + .fetch_all(&pool) + .await + .expect("stats"); + let covers: Vec<(Uuid, Uuid)> = sqlx::query_as( + "SELECT id, file_id FROM faces.faces WHERE user_id = $1 AND id = ANY($2)", + ) + .bind(seeded.user_id) + .bind(&seeded.cover_ids) + .fetch_all(&pool) + .await + .expect("covers"); + bytes = (stats.len() + covers.len()) * 32; + assert_eq!(stats.len(), persons); + } + times.push(t.elapsed().as_secs_f64() * 1000.0); + } + let ms = median(times); + let speedup = base + .map(|b: f64| format!("({:.1}x)", b / ms)) + .unwrap_or_default(); + println!("{mode:<28} {ms:>12.2} {bytes:>14} {speedup}"); + if base.is_none() { + base = Some(ms); + } + } + + cleanup(&pool, &seeded).await; +} diff --git a/examples/bench_propfind_paging.rs b/examples/bench_propfind_paging.rs new file mode 100644 index 00000000..012b1f72 --- /dev/null +++ b/examples/bench_propfind_paging.rs @@ -0,0 +1,233 @@ +//! PROPFIND folder-listing pagination benchmark — LIMIT/OFFSET vs keyset. +//! +//! The streaming PROPFIND walker pages a folder's children 500 at a time in +//! name order (`list_files_batch`). The old shape was `ORDER BY name LIMIT +//! 500 OFFSET k` with no supporting index — every page bitmap-scanned all N +//! children and top-sorted them, so a full folder walk was O(N²/500) row +//! visits. The change adds `idx_files_folder_name (folder_id, name) WHERE +//! NOT is_trashed` and switches the cursor to keyset (`name > $last`), making +//! each page one O(page) index-range read. +//! +//! Modes (full walk of the folder, all pages): +//! OFFSET/no-idx — the true BEFORE (index dropped for the run) +//! OFFSET/idx — index alone, old query shape +//! KEYSET/idx — the AFTER +//! +//! Run (needs Postgres up; reads DATABASE_URL from .env): +//! cargo run --release --features bench --example bench_propfind_paging +//! Tunables: BENCH_FILES (20000), BENCH_PAGE (500), BENCH_REPS (3) + +use std::env; +use std::time::Instant; + +use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; +use uuid::Uuid; + +fn env_or(key: &str, default: T) -> T { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +async fn seed(pool: &PgPool, files: usize) -> (Uuid, Uuid) { + let mut tx = pool.begin().await.expect("begin"); + let drive_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id", + ) + .fetch_one(&mut *tx) + .await + .expect("drive"); + let folder_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.folders (name, path, lpath, drive_id) + VALUES ('bench_paging', '/bench_paging', 'bench_paging', $1) RETURNING id", + ) + .bind(drive_id) + .fetch_one(&mut *tx) + .await + .expect("folder"); + sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2") + .bind(folder_id) + .bind(drive_id) + .execute(&mut *tx) + .await + .expect("stamp"); + tx.commit().await.expect("commit"); + + sqlx::query( + "INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id) + SELECT 'file_' || LPAD(i::text, 8, '0') || '.jpg', $1, + 'benchpaging00000000000000000000000000000000000000000000000000000', + 1024, 'image/jpeg', $2 + FROM generate_series(1, $3) AS i", + ) + .bind(folder_id) + .bind(drive_id) + .bind(files as i32) + .execute(pool) + .await + .expect("files"); + sqlx::query("ANALYZE storage.files") + .execute(pool) + .await + .ok(); + (drive_id, folder_id) +} + +const COLS: &str = "fi.id::text, fi.name, fi.folder_id::text, fo.path, fi.size, fi.mime_type, + EXTRACT(EPOCH FROM fi.created_at)::bigint, + EXTRACT(EPOCH FROM fi.updated_at)::bigint, fi.blob_hash"; + +type Row = ( + String, + String, + Option, + Option, + i64, + String, + i64, + i64, + String, +); + +/// Full folder walk with the old LIMIT/OFFSET shape. Returns rows seen. +async fn walk_offset(pool: &PgPool, folder: Uuid, page: i64) -> usize { + let mut offset = 0i64; + let mut seen = 0usize; + loop { + let rows: Vec = sqlx::query_as(&format!( + "SELECT {COLS} + FROM storage.files fi + LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + WHERE fi.folder_id = $1 AND NOT fi.is_trashed + ORDER BY fi.name LIMIT $2 OFFSET $3" + )) + .bind(folder) + .bind(page) + .bind(offset) + .fetch_all(pool) + .await + .expect("offset page"); + let n = rows.len(); + seen += n; + if (n as i64) < page { + break; + } + offset += n as i64; + } + seen +} + +/// Full folder walk with the new keyset shape. +async fn walk_keyset(pool: &PgPool, folder: Uuid, page: i64) -> usize { + let mut after: Option = None; + let mut seen = 0usize; + loop { + let rows: Vec = if let Some(a) = &after { + sqlx::query_as(&format!( + "SELECT {COLS} + FROM storage.files fi + LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + WHERE fi.folder_id = $1 AND NOT fi.is_trashed AND fi.name > $3 + ORDER BY fi.name LIMIT $2" + )) + .bind(folder) + .bind(page) + .bind(a) + .fetch_all(pool) + .await + } else { + sqlx::query_as(&format!( + "SELECT {COLS} + FROM storage.files fi + LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + WHERE fi.folder_id = $1 AND NOT fi.is_trashed + ORDER BY fi.name LIMIT $2" + )) + .bind(folder) + .bind(page) + .fetch_all(pool) + .await + } + .expect("keyset page"); + let n = rows.len(); + seen += n; + if (n as i64) < page { + break; + } + after = rows.last().map(|r| r.1.clone()); + } + seen +} + +fn median(mut xs: Vec) -> f64 { + xs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + xs[xs.len() / 2] +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + dotenvy::dotenv().ok(); + let url = env::var("DATABASE_URL").expect("set DATABASE_URL"); + let files: usize = env_or("BENCH_FILES", 20_000); + let page: i64 = env_or("BENCH_PAGE", 500); + let reps: usize = env_or("BENCH_REPS", 3); + + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(&url) + .await + .expect("connect"); + println!("seeding {files} files (one-time)…"); + let (drive_id, folder_id) = seed(&pool, files).await; + + println!("\n# full PROPFIND walk of a {files}-file folder, {page}/page"); + println!("{:<18} {:>12} {:>9}", "mode", "total ms", "vs OLD"); + + let mut base = None; + for mode in ["OFFSET/no-idx", "OFFSET/idx", "KEYSET/idx"] { + match mode { + "OFFSET/no-idx" => { + sqlx::query("DROP INDEX IF EXISTS storage.idx_files_folder_name") + .execute(&pool) + .await + .ok(); + } + "OFFSET/idx" => { + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_files_folder_name + ON storage.files (folder_id, name) WHERE NOT is_trashed", + ) + .execute(&pool) + .await + .expect("create index"); + } + _ => {} + } + let mut times = Vec::with_capacity(reps); + for _ in 0..reps { + let t = Instant::now(); + let seen = if mode.starts_with("OFFSET") { + walk_offset(&pool, folder_id, page).await + } else { + walk_keyset(&pool, folder_id, page).await + }; + assert_eq!(seen, files); + times.push(t.elapsed().as_secs_f64() * 1000.0); + } + let ms = median(times); + let speedup = base + .map(|b: f64| format!("{:.1}x", b / ms)) + .unwrap_or_else(|| "1.0x".into()); + if base.is_none() { + base = Some(ms); + } + println!("{mode:<18} {ms:>12.1} {speedup:>9}"); + } + + let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1") + .bind(drive_id) + .execute(&pool) + .await; +} diff --git a/examples/bench_quota_path.rs b/examples/bench_quota_path.rs new file mode 100644 index 00000000..0a520a20 --- /dev/null +++ b/examples/bench_quota_path.rs @@ -0,0 +1,169 @@ +//! Quota-path benchmark — full `auth.users` row vs narrow 2-column read. +//! +//! `check_storage_quota` (every upload) and `get_user_storage_info` (every +//! quota-reporting PROPFIND) used to call `get_user_by_id`, whose SELECT +//! drags the whole user row — including `image`, an avatar data URI of up +//! to 512 KiB — across the wire to read two i64s. The change reads only +//! `(storage_used_bytes, storage_quota_bytes)` +//! (`UserPgRepository::get_storage_usage`). Companion change measured here +//! as "SKIP": PROPFINDs whose prop list never names a quota prop now skip +//! the resolution entirely (`PropFindRequest::wants_quota`). +//! +//! Run (needs Postgres up; reads DATABASE_URL from .env): +//! cargo run --release --features bench --example bench_quota_path +//! Tunables: BENCH_SECONDS (4), BENCH_CONCURRENCIES ("8,64"), BENCH_IMAGE_KB (512) + +use std::env; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; +use uuid::Uuid; + +fn env_or(key: &str, default: T) -> T { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +async fn seed(pool: &PgPool, image_kb: usize) -> Uuid { + // Realistic worst-ish case: an avatar data URI at the documented cap. + let image = format!("data:image/png;base64,{}", "A".repeat(image_kb * 1024 - 22)); + sqlx::query_scalar( + "INSERT INTO auth.users (username, email, role, image) + VALUES ('bench_quota', 'bench_quota@bench.invalid', 'user', $1) + RETURNING id", + ) + .bind(&image) + .fetch_one(pool) + .await + .expect("seed user") +} + +async fn cleanup(pool: &PgPool, user_id: Uuid) { + let _ = sqlx::query("DELETE FROM auth.users WHERE id = $1") + .bind(user_id) + .execute(pool) + .await; +} + +/// BEFORE: the full-row SELECT `get_user_by_id` runs (same column list). +async fn one_op_full(pool: &PgPool, id: Uuid) { + let _row = sqlx::query( + r#" + SELECT + id, username, email, password_hash, role::text as role_text, + storage_quota_bytes, storage_used_bytes, + created_at, updated_at, last_login_at, active, + oidc_provider, oidc_subject, image, is_external, + given_name, family_name, email_verified_at, preferred_locale, notify_on_share, + ui_preferences + FROM auth.users + WHERE id = $1 + "#, + ) + .bind(id) + .fetch_one(pool) + .await + .expect("full row"); +} + +/// AFTER: the narrow `get_storage_usage` SELECT. +async fn one_op_narrow(pool: &PgPool, id: Uuid) { + let _row: (i64, i64) = sqlx::query_as( + "SELECT storage_used_bytes, storage_quota_bytes FROM auth.users WHERE id = $1", + ) + .bind(id) + .fetch_one(pool) + .await + .expect("narrow row"); +} + +struct Stats { + rps: f64, + p50: f64, + p99: f64, +} + +fn summarize(mut lats: Vec, secs: u64) -> Stats { + lats.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let n = lats.len(); + let pct = |p: f64| { + if n == 0 { + 0.0 + } else { + lats[((n as f64 * p) as usize).min(n - 1)] + } + }; + Stats { + rps: n as f64 / secs as f64, + p50: pct(0.50), + p99: pct(0.99), + } +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + dotenvy::dotenv().ok(); + let url = env::var("DATABASE_URL").expect("set DATABASE_URL"); + let secs: u64 = env_or("BENCH_SECONDS", 4); + let image_kb: usize = env_or("BENCH_IMAGE_KB", 512); + let concurrencies: Vec = env::var("BENCH_CONCURRENCIES") + .ok() + .map(|s| s.split(',').filter_map(|x| x.trim().parse().ok()).collect()) + .unwrap_or_else(|| vec![8, 64]); + + let pool = Arc::new( + PgPoolOptions::new() + .max_connections(20) + .min_connections(20) + .acquire_timeout(Duration::from_secs(10)) + .connect(&url) + .await + .expect("connect"), + ); + let user_id = seed(&pool, image_kb).await; + + println!("\n# quota lookup: full user row (incl. {image_kb} KiB avatar) vs 2-column read"); + println!( + "| {:>5} | {:<7} | {:>10} | {:>9} | {:>9} |", + "conc", "mode", "ops/s", "p50 µs", "p99 µs" + ); + for &conc in &concurrencies { + for mode in ["FULL", "NARROW"] { + let deadline = Instant::now() + Duration::from_secs(secs); + let mut handles = Vec::new(); + for _ in 0..conc { + let pool = pool.clone(); + let mode = mode.to_string(); + handles.push(tokio::spawn(async move { + let mut lats = Vec::new(); + while Instant::now() < deadline { + let t = Instant::now(); + if mode == "FULL" { + one_op_full(&pool, user_id).await; + } else { + one_op_narrow(&pool, user_id).await; + } + lats.push(t.elapsed().as_secs_f64() * 1e6); + } + lats + })); + } + let mut all = Vec::new(); + for h in handles { + all.extend(h.await.unwrap()); + } + let s = summarize(all, secs); + println!( + "| {:>5} | {:<7} | {:>10.0} | {:>9.1} | {:>9.1} |", + conc, mode, s.rps, s.p50, s.p99 + ); + } + } + println!("\n(SKIP: PROPFINDs not naming quota props now issue NEITHER query — 0 round-trips.)"); + + cleanup(&pool, user_id).await; +} diff --git a/examples/bench_static_precompress.rs b/examples/bench_static_precompress.rs new file mode 100644 index 00000000..9483503a --- /dev/null +++ b/examples/bench_static_precompress.rs @@ -0,0 +1,170 @@ +//! Static-asset compression benchmark — on-the-fly Brotli per request vs +//! serving a precompressed sibling. +//! +//! The SPA router compressed every compressible static response on the fly +//! (tower-http `CompressionLayer`, backed by `async-compression`'s Brotli at +//! `Level::Default`) — the same immutable `/_app/immutable` bundle re-encoded +//! on EVERY request. The change teaches `ServeDir` to serve build-time +//! `.br`/`.gz` siblings (`precompressed_br()/precompressed_gzip()` + +//! `frontend/scripts/precompress.mjs`), so a request costs a file read. +//! +//! This isolates exactly that per-request delta on a JS-bundle-like payload: +//! BEFORE — Brotli-encode the asset with async-compression Level::Default +//! (what the layer does per request) +//! AFTER — read the precompressed sibling from disk (what ServeDir does) +//! +//! Run (no Postgres needed): +//! cargo run --release --features bench --example bench_static_precompress +//! Tunables: BENCH_ASSET_KB (700), BENCH_REPS (30) + +use std::env; +use std::io::Write as _; +use std::time::Instant; + +use tokio::io::AsyncReadExt; + +fn env_or(key: &str, default: T) -> T { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// JS-like corpus: repetitive identifiers + literals, compresses like a real +/// minified bundle (roughly 3-5×). +fn synth_js(len: usize, seed: &mut u64) -> Vec { + const FRAGS: &[&str] = &[ + "function(e,t,n){var r=this;", + "return Object.assign({},", + "const a=document.querySelector(", + "export default{data(){return{", + "await fetch(url,{method:'POST',headers:", + ".map(function(x){return x.id});", + "if(void 0!==e&&null!==t){", + "console.error('unhandled',err);", + ]; + let mut out = Vec::with_capacity(len); + while out.len() < len { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + out.extend_from_slice(FRAGS[(*seed as usize) % FRAGS.len()].as_bytes()); + // sprinkle some varying identifiers so it's not pathological + let _ = write!(out, "v{}", *seed % 1000); + } + out.truncate(len); + out +} + +fn median(mut xs: Vec) -> f64 { + xs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + xs[xs.len() / 2] +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + let asset_kb: usize = env_or("BENCH_ASSET_KB", 700); + let reps: usize = env_or("BENCH_REPS", 30); + let mut seed = 0xC0FFEEu64; + let asset = synth_js(asset_kb * 1024, &mut seed); + + // Precompress once (build-time cost, paid once per deploy). + let dir = tempfile::tempdir().expect("tempdir"); + let br_path = dir.path().join("bundle.js.br"); + let t = Instant::now(); + let precompressed = { + use async_compression::tokio::bufread::BrotliEncoder; + let mut enc = BrotliEncoder::new(std::io::Cursor::new(asset.clone())); + let mut out = Vec::new(); + enc.read_to_end(&mut out).await.expect("precompress"); + out + }; + let build_ms = t.elapsed().as_secs_f64() * 1000.0; + std::fs::write(&br_path, &precompressed).expect("write .br"); + + println!( + "asset: {} KiB JS-like → {} KiB brotli ({}% smaller); one-time build cost {:.1} ms\n", + asset.len() / 1024, + precompressed.len() / 1024, + 100 - precompressed.len() * 100 / asset.len(), + build_ms + ); + + // BEFORE: per-request Brotli at the layer's default level. + let mut enc_times = Vec::with_capacity(reps); + for _ in 0..reps { + let t = Instant::now(); + use async_compression::tokio::bufread::BrotliEncoder; + let mut enc = BrotliEncoder::new(std::io::Cursor::new(asset.clone())); + let mut out = Vec::new(); + enc.read_to_end(&mut out).await.expect("encode"); + std::hint::black_box(&out); + enc_times.push(t.elapsed().as_secs_f64() * 1000.0); + } + + // AFTER: per-request read of the precompressed sibling. + let mut read_times = Vec::with_capacity(reps); + for _ in 0..reps { + let t = Instant::now(); + let mut f = tokio::fs::File::open(&br_path).await.expect("open"); + let mut out = Vec::new(); + f.read_to_end(&mut out).await.expect("read"); + std::hint::black_box(&out); + read_times.push(t.elapsed().as_secs_f64() * 1000.0); + } + + // ── Dynamic-response level sweep ───────────────────────────────────── + // The global API CompressionLayer (main.rs) compresses JSON responses + // per request. async-compression's Level::Default for Brotli is + // QUALITY 11 (brotli-8.0.2 encode.rs:323 via compression-codecs) — a + // deploy-grade setting on a per-request path. Sweep levels on a + // JSON-like 64 KiB body to pick the runtime quality. + let json_body = synth_js(64 * 1024, &mut seed); // JSON compresses like JS + println!("\n# per-request Brotli level on a 64 KiB JSON-like API response"); + println!("{:<22} {:>10} {:>12}", "level", "ms/resp", "out KiB"); + for (label, level) in [ + ("Default (= q11!)", async_compression::Level::Default), + ("Precise(4)", async_compression::Level::Precise(4)), + ("Fastest", async_compression::Level::Fastest), + ] { + let mut times = Vec::with_capacity(reps); + let mut out_len = 0; + for _ in 0..reps { + let t = Instant::now(); + use async_compression::tokio::bufread::BrotliEncoder; + let mut enc = + BrotliEncoder::with_quality(std::io::Cursor::new(json_body.clone()), level); + let mut out = Vec::new(); + enc.read_to_end(&mut out).await.expect("encode"); + out_len = out.len(); + std::hint::black_box(&out); + times.push(t.elapsed().as_secs_f64() * 1000.0); + } + println!( + "{:<22} {:>10.2} {:>12.1}", + label, + median(times), + out_len as f64 / 1024.0 + ); + } + + let enc = median(enc_times); + let read = median(read_times); + println!( + "{:<34} {:>10} {:>9}", + "mode (per request)", "ms", "vs BEFORE" + ); + println!( + "{:<34} {:>10.2} {:>9}", + "BEFORE on-the-fly Brotli", enc, "1.0x" + ); + println!( + "{:<34} {:>10.3} {:>8.0}x", + "AFTER precompressed read", + read, + enc / read + ); + println!("\n(BEFORE also holds ~1 tokio task busy for the duration on every request;"); + println!(" AFTER additionally ships the deploy-time q11 encoding, usually smaller than"); + println!(" the runtime default level.)"); +} diff --git a/examples/bench_zip_media.rs b/examples/bench_zip_media.rs new file mode 100644 index 00000000..dd60ca81 --- /dev/null +++ b/examples/bench_zip_media.rs @@ -0,0 +1,262 @@ +//! ZIP entry-compression benchmark — `Deflate`-always vs MIME-aware `Stored`. +//! +//! Isolates the ONE variable the ZIP-export change touches: the per-entry +//! `Compression` mode chosen by `ZipService::write_prefetched_file` / +//! `BatchOperations::add_file_entry_streamed`. It rebuilds the *exact* +//! production writer stack — +//! +//! `ZipFileWriter::with_tokio(BufWriter(File))` + `write_entry_stream` +//! fed in ~64 KiB chunks (the blob-stream chunk size) +//! +//! — and writes the same corpus once per mode, measuring wall time, process +//! CPU time (utime+stime from `/proc/self/stat`), and final archive size. +//! +//! Corpora: +//! • `media` — incompressible bytes (models JPEG/HEIC/MP4/WebP, the +//! dominant "download folder" payload). Deflate here is pure CPU burn. +//! • `text` — compressible text (models docs/source). Deflate genuinely +//! shrinks these; the MIME-aware change keeps deflating them. +//! • `mixed` — 80 % media / 20 % text by bytes: `all-Deflate` row is the +//! production behaviour BEFORE the change; `mime-aware` row (Stored for +//! media, Deflate for text) is AFTER. +//! +//! Run (no Postgres needed): +//! cargo run --release --features bench --example bench_zip_media +//! Tunables (env): +//! BENCH_MEDIA_FILES (48) BENCH_MEDIA_MB (4) per-file size +//! BENCH_TEXT_FILES (24) BENCH_TEXT_MB (2) +//! BENCH_REPS (3) median reported + +use std::env; +use std::time::{Duration, Instant}; + +use async_zip::base::write::ZipFileWriter; +use async_zip::{Compression, ZipEntryBuilder}; +use futures::io::AsyncWriteExt as FuturesWriteExt; +use tokio::io::BufWriter; + +const CHUNK: usize = 64 * 1024; // blob-stream chunk size on the real path + +fn env_or(key: &str, default: T) -> T { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +/// Process CPU seconds (user + system) from /proc/self/stat — covers all +/// threads, so it catches deflate work wherever tokio schedules it. +fn cpu_seconds() -> f64 { + let stat = std::fs::read_to_string("/proc/self/stat").expect("read /proc/self/stat"); + // utime and stime are fields 14 and 15 (1-based), after the comm field + // which may contain spaces — skip past the closing paren first. + let after = &stat[stat.rfind(')').unwrap() + 2..]; + let fields: Vec<&str> = after.split_whitespace().collect(); + let utime: u64 = fields[11].parse().unwrap(); // field 14 overall + let stime: u64 = fields[12].parse().unwrap(); // field 15 overall + (utime + stime) as f64 / 100.0 // USER_HZ = 100 on Linux +} + +/// Deterministic xorshift64* stream — incompressible "media" bytes. +fn fill_random(buf: &mut [u8], seed: &mut u64) { + for chunk in buf.chunks_mut(8) { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + let bytes = seed.wrapping_mul(0x2545F4914F6CDD1D).to_le_bytes(); + let n = chunk.len(); + chunk.copy_from_slice(&bytes[..n]); + } +} + +/// Compressible pseudo-text (~3-4× deflate ratio, like real docs/source). +fn fill_text(buf: &mut [u8], seed: &mut u64) { + const WORDS: &[&str] = &[ + "the", + "quick", + "brown", + "fox", + "jumps", + "over", + "lazy", + "dog", + "folder", + "file", + "storage", + "performance", + "benchmark", + "archive", + "download", + "stream", + ]; + let mut pos = 0; + while pos < buf.len() { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + let w = WORDS[(*seed as usize) % WORDS.len()].as_bytes(); + let n = w.len().min(buf.len() - pos); + buf[pos..pos + n].copy_from_slice(&w[..n]); + pos += n; + if pos < buf.len() { + buf[pos] = b' '; + pos += 1; + } + } +} + +struct CorpusFile { + name: String, + data: Vec, + is_media: bool, +} + +struct RunResult { + wall: Duration, + cpu: f64, + bytes_out: u64, +} + +/// Write the corpus through the exact production writer stack, choosing the +/// compression mode per entry with `pick`. +async fn write_zip(files: &[CorpusFile], pick: impl Fn(&CorpusFile) -> Compression) -> RunResult { + let temp = tempfile::NamedTempFile::new().expect("temp file"); + let tokio_file = tokio::fs::File::create(temp.path()).await.expect("create"); + let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file); + let mut zip = ZipFileWriter::with_tokio(buf_writer); + + let cpu0 = cpu_seconds(); + let t0 = Instant::now(); + for f in files { + let entry = ZipEntryBuilder::new(f.name.clone().into(), pick(f)); + let mut w = zip.write_entry_stream(entry).await.expect("entry start"); + for chunk in f.data.chunks(CHUNK) { + w.write_all(chunk).await.expect("chunk write"); + } + w.close().await.expect("entry close"); + } + let mut compat = zip.close().await.expect("zip close"); + compat.close().await.expect("flush"); + let wall = t0.elapsed(); + let cpu = cpu_seconds() - cpu0; + + let bytes_out = std::fs::metadata(temp.path()).map(|m| m.len()).unwrap_or(0); + RunResult { + wall, + cpu, + bytes_out, + } +} + +fn median(mut xs: Vec) -> f64 { + xs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + xs[xs.len() / 2] +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + let media_files: usize = env_or("BENCH_MEDIA_FILES", 48); + let media_mb: usize = env_or("BENCH_MEDIA_MB", 4); + let text_files: usize = env_or("BENCH_TEXT_FILES", 24); + let text_mb: usize = env_or("BENCH_TEXT_MB", 2); + let reps: usize = env_or("BENCH_REPS", 3); + + let mut seed = 0x9E3779B97F4A7C15u64; + let mut corpus: Vec = Vec::new(); + for i in 0..media_files { + let mut data = vec![0u8; media_mb * 1024 * 1024]; + fill_random(&mut data, &mut seed); + corpus.push(CorpusFile { + name: format!("photos/IMG_{i:04}.jpg"), + data, + is_media: true, + }); + } + for i in 0..text_files { + let mut data = vec![0u8; text_mb * 1024 * 1024]; + fill_text(&mut data, &mut seed); + corpus.push(CorpusFile { + name: format!("docs/notes_{i:04}.txt"), + data, + is_media: false, + }); + } + let media_bytes: usize = corpus + .iter() + .filter(|f| f.is_media) + .map(|f| f.data.len()) + .sum(); + let text_bytes: usize = corpus + .iter() + .filter(|f| !f.is_media) + .map(|f| f.data.len()) + .sum(); + let total_mb = (media_bytes + text_bytes) as f64 / 1048576.0; + println!( + "corpus: {} media files ({} MiB, incompressible) + {} text files ({} MiB, compressible), {} reps\n", + media_files, + media_bytes / 1048576, + text_files, + text_bytes / 1048576, + reps + ); + + // (label, per-entry compression picker) + type Picker = Box Compression>; + let modes: Vec<(&str, Picker)> = vec![ + ( + "all-Deflate (BEFORE)", + Box::new(|_: &CorpusFile| Compression::Deflate), + ), + ( + "mime-aware (AFTER) ", + Box::new(|f: &CorpusFile| { + if f.is_media { + Compression::Stored + } else { + Compression::Deflate + } + }), + ), + ( + "all-Stored (bound) ", + Box::new(|_: &CorpusFile| Compression::Stored), + ), + ]; + + println!( + "{:<22} {:>9} {:>9} {:>10} {:>11} {:>9}", + "mode", "wall s", "cpu s", "MB/s", "out MiB", "ratio" + ); + let mut baseline_wall = None; + for (label, pick) in &modes { + let mut walls = Vec::new(); + let mut cpus = Vec::new(); + let mut out = 0u64; + for _ in 0..reps { + let r = write_zip(&corpus, pick).await; + walls.push(r.wall.as_secs_f64()); + cpus.push(r.cpu); + out = r.bytes_out; + } + let wall = median(walls); + let cpu = median(cpus); + let speedup = baseline_wall + .map(|b: f64| format!("{:.2}x", b / wall)) + .unwrap_or_else(|| "1.00x".into()); + if baseline_wall.is_none() { + baseline_wall = Some(wall); + } + println!( + "{:<22} {:>9.3} {:>9.2} {:>10.1} {:>11.1} {:>9}", + label, + wall, + cpu, + total_mb / wall, + out as f64 / 1048576.0, + speedup + ); + } + println!("\n(archive `out MiB` for mime-aware stays ~= all-Deflate: media doesn't deflate,"); + println!(" text keeps Deflate — the win is CPU/wall, not size loss)"); +} diff --git a/frontend/package.json b/frontend/package.json index 917c16a9..9d2974fa 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,7 +9,7 @@ "scripts": { "dev": "vite dev", "build": "vite build", - "postbuild": "node scripts/emit-askama-common.mjs", + "postbuild": "node scripts/emit-askama-common.mjs && node scripts/precompress.mjs ../static-dist", "preview": "vite preview", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json && eslint . && stylelint \"src/**/*.{css,svelte}\" && prettier --check .", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", diff --git a/frontend/scripts/precompress.mjs b/frontend/scripts/precompress.mjs new file mode 100644 index 00000000..3e3fddf6 --- /dev/null +++ b/frontend/scripts/precompress.mjs @@ -0,0 +1,60 @@ +// Precompress built SPA assets so the Rust web layer can serve them with +// `ServeDir::precompressed_br()/precompressed_gzip()` instead of re-running +// Brotli over the same immutable bundle on every request (the tower-http +// CompressionLayer stays as the on-the-fly fallback for anything without a +// sibling). Runs as the `build` script's final step; uses only node:zlib — +// no dependencies. See benches/STATIC-PRECOMPRESSED.md for the measured win. +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import zlib from 'node:zlib'; + +const OUT_DIR = process.argv[2] ?? '../static-dist'; +// Compressible text assets; media formats are already compressed. +const EXTENSIONS = new Set([ + '.js', + '.mjs', + '.css', + '.html', + '.svg', + '.json', + '.txt', + '.xml', + '.map', + '.webmanifest' +]); +// Below this size the encoding overhead outweighs the transfer win +// (mirrors the server's SizeAbove(256) predicate). +const MIN_BYTES = 256; + +async function* walk(dir) { + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) yield* walk(p); + else yield p; + } +} + +let files = 0; +let inBytes = 0; +let brBytes = 0; +for await (const file of walk(OUT_DIR)) { + if (!EXTENSIONS.has(path.extname(file))) continue; + const data = await fs.readFile(file); + if (data.length < MIN_BYTES) continue; + const br = zlib.brotliCompressSync(data, { + params: { + [zlib.constants.BROTLI_PARAM_QUALITY]: 11, + [zlib.constants.BROTLI_PARAM_SIZE_HINT]: data.length + } + }); + const gz = zlib.gzipSync(data, { level: 9 }); + // Only keep siblings that actually shrink the asset. + if (br.length < data.length) await fs.writeFile(`${file}.br`, br); + if (gz.length < data.length) await fs.writeFile(`${file}.gz`, gz); + files += 1; + inBytes += data.length; + brBytes += Math.min(br.length, data.length); +} +console.log( + `precompress: ${files} assets, ${(inBytes / 1024).toFixed(0)} KiB → ${(brBytes / 1024).toFixed(0)} KiB brotli (${inBytes ? ((1 - brBytes / inBytes) * 100).toFixed(0) : 0}% smaller)` +); diff --git a/frontend/src/lib/api/endpoints/folders.ts b/frontend/src/lib/api/endpoints/folders.ts index 2bc8adee..b88af15c 100644 --- a/frontend/src/lib/api/endpoints/folders.ts +++ b/frontend/src/lib/api/endpoints/folders.ts @@ -108,7 +108,19 @@ export async function getFolder(id: string): Promise { */ export async function fetchFolderListing( folderId: string, - opts: { etag?: string; forceRefresh?: boolean } = {} + opts: { + etag?: string; + forceRefresh?: boolean; + /** + * Progressive render hook: invoked after EVERY page with the + * accumulated listing so far (the arrays are fresh copies — safe to + * hand to reactive state). Without it, a 2,000-item folder waited + * for all ⌈N/200⌉ sequential round-trips before the first row + * painted; with it the view paints after page one (~200 items) and + * fills in as the tail pages land. + */ + onPage?: (partial: FolderListing, done: boolean) => void; + } = {} ): Promise { const folders: FolderItem[] = []; const files: FileItem[] = []; @@ -132,6 +144,10 @@ export async function fetchFolderListing( else files.push(it.resource as FileItem); } cursor = page.next_cursor; + opts.onPage?.( + { folders: [...folders], files: [...files], favoriteIds: [], sharedIds: [] }, + !cursor + ); } while (cursor); return { status: 200, listing: { folders, files, favoriteIds: [], sharedIds: [] } }; diff --git a/frontend/src/routes/files/[...path]/+page.svelte b/frontend/src/routes/files/[...path]/+page.svelte index da318aac..51ce1d5a 100644 --- a/frontend/src/routes/files/[...path]/+page.svelte +++ b/frontend/src/routes/files/[...path]/+page.svelte @@ -308,7 +308,22 @@ }); try { - const res = await fetchFolderListing(folderId, { etag: cached?.etag }); + const res = await fetchFolderListing(folderId, { + etag: cached?.etag, + // Paint page one (~200 items) immediately instead of waiting + // for every sequential page of a large folder; later pages + // extend the view as they land. Skip when a cached copy is + // already on screen — replacing it with a partial list would + // briefly shrink the view. + onPage: cached + ? undefined + : (partial, done) => { + if (seq !== loadSeq || done) return; // final state applied below + applyListing(partial); + loading = false; + showSkeleton = false; + } + }); if (seq !== loadSeq) return; // superseded by a newer navigation if (res.status === 200 && res.listing) { applyListing(res.listing); diff --git a/migrations/20260917000000_files_folder_name_index.sql b/migrations/20260917000000_files_folder_name_index.sql new file mode 100644 index 00000000..9241840d --- /dev/null +++ b/migrations/20260917000000_files_folder_name_index.sql @@ -0,0 +1,22 @@ +-- ════════════════════════════════════════════════════════════════════════════ +-- Name-ordered folder listing index for streaming WebDAV PROPFIND +-- ════════════════════════════════════════════════════════════════════════════ +-- `list_files_batch` walks a folder's children in `ORDER BY name` pages of +-- 500 (native + NextCloud PROPFIND). The only index on the filter column +-- was `idx_files_folder_id (folder_id)`, so EVERY page did a bitmap scan of +-- all N children plus a top-(offset+limit) sort — a quadratic full-folder +-- walk (the initial schema's `(folder_id, name, user_id)` index that served +-- this was dropped by 20260902000000 when user_id went nullable). +-- +-- This composite index restores the ordered access path: combined with the +-- keyset cursor (`name > $last` — see `file_blob_read_repository.rs` +-- `list_files_batch`), each page is one O(page) index-range read with no +-- sort, regardless of folder size or scroll depth. Benchmarked in +-- benches/DEAD-PROPS.md's companion doc benches/PROPFIND-PAGING.md. +-- +-- Partial (`NOT is_trashed`) to match the listing predicate and keep the +-- index compact; trashed rows are never listed by PROPFIND. + +CREATE INDEX IF NOT EXISTS idx_files_folder_name + ON storage.files (folder_id, name) + WHERE NOT is_trashed; diff --git a/src/application/adapters/webdav_adapter.rs b/src/application/adapters/webdav_adapter.rs index ce03ffc5..05a58613 100644 --- a/src/application/adapters/webdav_adapter.rs +++ b/src/application/adapters/webdav_adapter.rs @@ -129,6 +129,30 @@ pub struct PropFindRequest { pub prop_find_type: PropFindType, } +impl PropFindRequest { + /// Whether answering this PROPFIND requires resolving the account / + /// drive quota at all. + /// + /// `resolve_webdav_quota` costs two DB round-trips per request; sync + /// clients poll folders with an explicit `` list that most of + /// the time names only etag/length/type props — computing quota there + /// is pure waste (the response never mentions it). `AllProp` and + /// `PropName` keep quota: the writers emit RFC 4331 props for both. + /// Measured in `benches/QUOTA-PATH.md`. + pub fn wants_quota(&self) -> bool { + match &self.prop_find_type { + PropFindType::AllProp | PropFindType::PropName => true, + PropFindType::Prop(props) => props.iter().any(|p| { + p.namespace == "DAV:" + && matches!( + p.name.as_str(), + "quota-used-bytes" | "quota-available-bytes" + ) + }), + } + } +} + /// WebDAV property value #[derive(Debug, Clone)] pub struct PropValue { diff --git a/src/application/ports/authorization_ports.rs b/src/application/ports/authorization_ports.rs index 6a02f696..fb994362 100644 --- a/src/application/ports/authorization_ports.rs +++ b/src/application/ports/authorization_ports.rs @@ -29,6 +29,30 @@ pub trait AuthorizationEngine: Send + Sync + 'static { resource: Resource, ) -> Result; + /// Batched `check(subject, Read, File(id))` over a result page: returns + /// the subset of `file_ids` the subject may read. Semantically identical + /// to looping [`Self::check`] (the default does exactly that); the + /// `PgAclEngine` override resolves every file's drive in ONE query and + /// reuses the per-drive role cache, so verifying a 200-hit search page + /// costs 1 SQL round-trip instead of up to 200 sequential ones + /// (benches/SEARCH-REBAC.md). + async fn check_files_read_batch( + &self, + subject: Subject, + file_ids: &[Uuid], + ) -> Result, DomainError> { + let mut allowed = std::collections::HashSet::with_capacity(file_ids.len()); + for id in file_ids { + if self + .check(subject, Permission::Read, Resource::File(*id)) + .await? + { + allowed.insert(*id); + } + } + Ok(allowed) + } + /// Convenience wrapper around `check`: returns `Ok(())` when allowed and /// `DomainError::not_found` when denied (anti-enumeration — same error as /// "resource doesn't exist" so attackers can't probe IDs by error shape). diff --git a/src/application/ports/face_ports.rs b/src/application/ports/face_ports.rs index 95e00eb8..5dd00fe4 100644 --- a/src/application/ports/face_ports.rs +++ b/src/application/ports/face_ports.rs @@ -39,6 +39,23 @@ pub trait FaceRepository: Send + Sync + 'static { user_id: Uuid, blob_hash: &str, ) -> Result, DomainError>; + /// `(person_id, face_count)` per non-empty cluster — a grouped COUNT + /// instead of dragging every face row (each with a 2 KiB embedding + /// BYTEA) across the wire just to count them. See benches/PEOPLE-LIST.md. + async fn person_face_stats(&self, user_id: Uuid) -> Result, DomainError>; + /// face id → file id for the given faces (cover-photo resolution). + async fn file_ids_for_faces( + &self, + user_id: Uuid, + face_ids: &[Uuid], + ) -> Result, DomainError>; + /// Reassign every face of `from` to `into` in one statement (merge). + async fn reassign_person_faces( + &self, + user_id: Uuid, + from: Uuid, + into: Uuid, + ) -> Result; async fn assign_person( &self, face_id: Uuid, diff --git a/src/application/ports/file_ports.rs b/src/application/ports/file_ports.rs index f4920106..e2f6b064 100644 --- a/src/application/ports/file_ports.rs +++ b/src/application/ports/file_ports.rs @@ -235,13 +235,14 @@ pub trait FileRetrievalUseCase: Send + Sync + 'static { async fn list_files_batch( &self, folder_id: Option<&str>, - offset: i64, + after_name: Option<&str>, limit: i64, ) -> Result, DomainError> { - let all = self.list_files(folder_id).await?; + let mut all = self.list_files(folder_id).await?; + all.sort_by(|a, b| a.name.cmp(&b.name)); Ok(all .into_iter() - .skip(offset as usize) + .filter(|f| after_name.is_none_or(|a| f.name.as_str() > a)) .take(limit as usize) .collect()) } @@ -257,10 +258,10 @@ pub trait FileRetrievalUseCase: Send + Sync + 'static { &self, folder_id: Option<&str>, _owner_id: Uuid, - offset: i64, + after_name: Option<&str>, limit: i64, ) -> Result, DomainError> { - self.list_files_batch(folder_id, offset, limit).await + self.list_files_batch(folder_id, after_name, limit).await } } diff --git a/src/application/ports/share_ports.rs b/src/application/ports/share_ports.rs index f6b6d151..5e7600d2 100644 --- a/src/application/ports/share_ports.rs +++ b/src/application/ports/share_ports.rs @@ -99,6 +99,28 @@ pub trait ShareStoragePort: Send + Sync + 'static { share: &crate::domain::entities::share::Share, ) -> Result; + /// Atomically bump a link's access counter (public share landing). + /// Returns the number of rows updated — 0 means "no live share for + /// this token" (missing OR expired). + /// + /// The default is the legacy read-modify-write (kept for test mocks); + /// `SharePgRepository` overrides it with a single `UPDATE … SET + /// access_count = access_count + 1`, replacing 2 correlated-subquery + /// round-trips per anonymous visit with 1 and removing the lost-update + /// race between concurrent visitors (benches/SHARE-ACCESS.md). + async fn increment_access_count(&self, token: &str) -> Result { + let share = match self.find_share_by_token(token).await { + Ok(s) => s, + Err(e) if e.kind == crate::common::errors::ErrorKind::NotFound => return Ok(0), + Err(e) => return Err(e), + }; + if share.is_expired() { + return Ok(0); + } + self.update_share(&share.increment_access_count()).await?; + Ok(1) + } + async fn find_shares_by_user( &self, user_id: Uuid, diff --git a/src/application/ports/storage_ports.rs b/src/application/ports/storage_ports.rs index 1157f649..896fc01c 100644 --- a/src/application/ports/storage_ports.rs +++ b/src/application/ports/storage_ports.rs @@ -107,20 +107,30 @@ pub trait FileReadPort: Send + Sync + 'static { Ok(None) } - /// Lists files in a folder with LIMIT/OFFSET pagination. + /// Lists files in a folder in name order, keyset-paginated. /// /// Used by streaming WebDAV PROPFIND to avoid loading all files at once. + /// `after_name` is the last name of the previous page (`None` = first + /// page); names are unique within a folder (unique index on + /// `(drive_id, folder_id, name)`), so `name > after_name` is a total, + /// stable cursor. Unlike LIMIT/OFFSET, every page is O(page) — the old + /// offset shape re-scanned and re-sorted the whole folder per page + /// (benches/PROPFIND-PAGING.md). + /// /// Default: falls back to `list_files` (loads all, then slices in memory). async fn list_files_batch( &self, folder_id: Option<&str>, - offset: i64, + after_name: Option<&str>, limit: i64, ) -> Result, DomainError> { - let all = self.list_files(folder_id).await?; - let start = (offset as usize).min(all.len()); - let end = (start + limit as usize).min(all.len()); - Ok(all.into_iter().skip(start).take(end - start).collect()) + let mut all = self.list_files(folder_id).await?; + all.sort_by(|a, b| a.name().cmp(b.name())); + Ok(all + .into_iter() + .filter(|f| after_name.is_none_or(|a| f.name() > a)) + .take(limit as usize) + .collect()) } /// Streams every file in the subtree rooted at `folder_id`. diff --git a/src/application/services/batch_operations.rs b/src/application/services/batch_operations.rs index 1329af88..bd75d2c5 100644 --- a/src/application/services/batch_operations.rs +++ b/src/application/services/batch_operations.rs @@ -734,7 +734,13 @@ impl BatchOperationService { { Ok(file_dto) => { match self - .add_file_entry_streamed(&mut zip, file_id, &file_dto.name, user_id) + .add_file_entry_streamed( + &mut zip, + file_id, + &file_dto.name, + &file_dto.mime_type, + Some(user_id), + ) .await { Ok(_) => items_added += 1, @@ -758,7 +764,7 @@ impl BatchOperationService { { Ok(root_folder) => { match self - .add_folder_subtree_to_zip(&mut zip, folder_id, &root_folder, user_id) + .add_folder_subtree_to_zip(&mut zip, folder_id, &root_folder) .await { Ok(_) => items_added += 1, @@ -803,24 +809,44 @@ impl BatchOperationService { } /// Streams a single file into an async ZIP entry (~64 KB peak RAM per file). + /// + /// Already-compressed content (per its MIME type) is `Stored` — deflating + /// JPEG/MP4/… burns ~a CPU core per download for ~0 % size gain. + /// + /// `caller_id = Some(uid)` enforces the per-file Read check and records + /// the access in Recents (explicitly-selected top-level files). + /// `None` = the file was enumerated from a folder subtree whose ROOT the + /// caller already passed `get_folder_with_perms` for — per-file + /// re-authorization and per-file Recent spam (2 writes/file via the + /// recent hook) are skipped, mirroring `ZipService::create_folder_zip` + /// on the native folder-download path (benches/ZIP-BATCH-AUTHZ.md). async fn add_file_entry_streamed( &self, zip: &mut ZipFileWriter>>, file_id: &str, entry_name: &str, - caller_id: Uuid, + mime_type: &str, + caller_id: Option, ) -> Result<(), BatchOperationError> { - let entry = ZipEntryBuilder::new(entry_name.to_string().into(), Compression::Deflate); + let compression = crate::common::mime_detect::zip_entry_compression(mime_type); + let entry = ZipEntryBuilder::new(entry_name.to_string().into(), compression); let mut writer = zip .write_entry_stream(entry) .await .map_err(|e| BatchOperationError::Internal(format!("zip entry start: {}", e)))?; - let stream = self - .file_retrieval - .get_file_stream_with_perms(file_id, caller_id) - .await - .map_err(BatchOperationError::Domain)?; + let stream = match caller_id { + Some(uid) => self + .file_retrieval + .get_file_stream_with_perms(file_id, uid) + .await + .map_err(BatchOperationError::Domain)?, + None => self + .file_retrieval + .get_file_stream(file_id) + .await + .map_err(BatchOperationError::Domain)?, + }; let mut stream = std::pin::Pin::from(stream); while let Some(chunk) = stream.next().await { @@ -849,7 +875,6 @@ impl BatchOperationService { zip: &mut ZipFileWriter>>, folder_id: &str, root_folder: &FolderDto, - caller_id: Uuid, ) -> Result<(), BatchOperationError> { // Bulk-fetch folder tree (small — one entry per folder) let all_folders = self @@ -903,8 +928,10 @@ impl BatchOperationService { if let Some(files) = files_by_folder.get(&folder.id) { for file in files { let file_path = format!("{}{}", zip_dir, file.name); + // Subtree pre-authorized at the root folder — see + // `add_file_entry_streamed` docs for why `None`. if let Err(e) = self - .add_file_entry_streamed(zip, &file.id, &file_path, caller_id) + .add_file_entry_streamed(zip, &file.id, &file_path, &file.mime_type, None) .await { info!("Could not add file {} to ZIP: {}", file.name, e); diff --git a/src/application/services/file_retrieval_service.rs b/src/application/services/file_retrieval_service.rs index 522c6acc..7109739b 100644 --- a/src/application/services/file_retrieval_service.rs +++ b/src/application/services/file_retrieval_service.rs @@ -451,12 +451,12 @@ impl FileRetrievalUseCase for FileRetrievalService { async fn list_files_batch( &self, folder_id: Option<&str>, - offset: i64, + after_name: Option<&str>, limit: i64, ) -> Result, DomainError> { let files = self .file_read - .list_files_batch(folder_id, offset, limit) + .list_files_batch(folder_id, after_name, limit) .await?; Ok(files.into_iter().map(FileDto::from).collect()) } @@ -465,7 +465,7 @@ impl FileRetrievalUseCase for FileRetrievalService { &self, folder_id: Option<&str>, owner_id: Uuid, - offset: i64, + after_name: Option<&str>, limit: i64, ) -> Result, DomainError> { // Post-D0: every file lives in a folder — `storage.files.folder_id` @@ -482,7 +482,7 @@ impl FileRetrievalUseCase for FileRetrievalService { .await?; let files = self .file_read - .list_files_batch(folder_id, offset, limit) + .list_files_batch(folder_id, after_name, limit) .await?; Ok(files.into_iter().map(FileDto::from).collect()) } diff --git a/src/application/services/people_service.rs b/src/application/services/people_service.rs index 6e3ac8b5..317caa63 100644 --- a/src/application/services/people_service.rs +++ b/src/application/services/people_service.rs @@ -166,18 +166,23 @@ impl PeopleService { } /// People (non-empty clusters), most-photographed first. + /// + /// Counts come from a grouped-COUNT query and cover photos from one + /// batched lookup of just the cover face ids — the previous + /// `faces_for_user` shipped every face row (2 KiB embedding included) + /// only to count them: ~20 MB of BYTEA per request on a 10k-face + /// library (benches/PEOPLE-LIST.md). pub async fn list_people(&self, caller_id: Uuid) -> Result, DomainError> { let persons = self.repo.persons_for_user(caller_id).await?; - let faces = self.repo.faces_for_user(caller_id).await?; - - let mut count: HashMap = HashMap::new(); - let mut face_file: HashMap = HashMap::new(); - for f in &faces { - if let Some(pid) = f.person_id { - *count.entry(pid).or_default() += 1; - } - face_file.insert(f.id, f.file_id); - } + let count: HashMap = self + .repo + .person_face_stats(caller_id) + .await? + .into_iter() + .collect(); + let cover_ids: Vec = persons.iter().filter_map(|p| p.cover_face_id).collect(); + let face_file: HashMap = + self.repo.file_ids_for_faces(caller_id, &cover_ids).await?; let mut out: Vec = persons .into_iter() @@ -245,11 +250,13 @@ impl PeopleService { /// Merge `from` into `into` by reassigning all of `from`'s faces. The /// now-empty `from` person is hidden by `list_people`. + /// + /// One set-based UPDATE — the previous shape loaded every face row + /// (embeddings included) and issued one UPDATE per matching face. pub async fn merge(&self, caller_id: Uuid, into: Uuid, from: Uuid) -> Result<(), DomainError> { - let faces = self.repo.faces_for_user(caller_id).await?; - for f in faces.into_iter().filter(|f| f.person_id == Some(from)) { - self.repo.assign_person(f.id, Some(into)).await?; - } + self.repo + .reassign_person_faces(caller_id, from, into) + .await?; Ok(()) } diff --git a/src/application/services/search_service.rs b/src/application/services/search_service.rs index 92a35e05..ff1ccd11 100644 --- a/src/application/services/search_service.rs +++ b/src/application/services/search_service.rs @@ -260,7 +260,7 @@ impl SearchService { user_id: Uuid, ) -> Vec { use crate::application::ports::authorization_ports::AuthorizationEngine; - use crate::domain::services::authorization::{Permission, Resource, Subject}; + use crate::domain::services::authorization::Subject; let Some(index) = &self.content_index else { return Vec::new(); @@ -316,36 +316,42 @@ impl SearchService { // drive the caller doesn't otherwise have. The Tantivy // filter is drive-only; this re-check restores per-file // resolution. - // Failures degrade conservatively (drop the hit, log it) — - // never leak. - let mut verified = Vec::with_capacity(hits.len()); - for hit in hits { - let file_uuid = match Uuid::parse_str(&hit.file_id) { - Ok(u) => u, + // Failures degrade conservatively (drop the hit / the page, + // log it) — never leak. Batched: one drive-resolution query for + // the whole page instead of up to CONTENT_HITS_LIMIT sequential + // point SELECTs (benches/SEARCH-REBAC.md). + let mut hit_ids = Vec::with_capacity(hits.len()); + for hit in &hits { + match Uuid::parse_str(&hit.file_id) { + Ok(u) => hit_ids.push(u), Err(_) => { tracing::warn!("Content-index hit had non-UUID file_id: {}", hit.file_id); - continue; } + } + } + let allowed = match authz + .check_files_read_batch(Subject::User(user_id), &hit_ids) + .await + { + Ok(set) => set, + Err(e) => { + tracing::warn!("ReBAC re-check failed for content hits: {e}"); + return Vec::new(); + } + }; + let mut verified = Vec::with_capacity(hits.len()); + for hit in hits { + let Ok(file_uuid) = Uuid::parse_str(&hit.file_id) else { + continue; // already warned above }; - match authz - .check( - Subject::User(user_id), - Permission::Read, - Resource::File(file_uuid), - ) - .await - { - Ok(true) => verified.push(hit), - Ok(false) => { - tracing::debug!( - target: "oxicloud::search", - file_id = %file_uuid, - "dropping content-index hit: ReBAC denies Read after Tantivy filter", - ); - } - Err(e) => { - tracing::warn!("ReBAC re-check failed for {file_uuid}: {e}"); - } + if allowed.contains(&file_uuid) { + verified.push(hit); + } else { + tracing::debug!( + target: "oxicloud::search", + file_id = %file_uuid, + "dropping content-index hit: ReBAC denies Read after Tantivy filter", + ); } } verified diff --git a/src/application/services/share_service.rs b/src/application/services/share_service.rs index 69c56039..cde8c43b 100644 --- a/src/application/services/share_service.rs +++ b/src/application/services/share_service.rs @@ -510,29 +510,18 @@ impl ShareUseCase for ShareService { } async fn register_shared_link_access(&self, token: &str) -> Result<(), DomainError> { - // Find the shared link by its token - let share = self - .share_repository - .find_share_by_token(token) - .await - .map_err(|e| { - ShareServiceError::NotFound(format!("Share with token {} not found: {}", token, e)) - })?; - - // Check if it has expired - if share.is_expired() { - return Err(ShareServiceError::Expired.into()); + // One atomic UPDATE (see `ShareStoragePort::increment_access_count`). + // 0 rows = missing or expired — collapsed into NotFound, same + // response shape either way (anti-enumeration; the landing handler + // discards this result regardless). + let updated = self.share_repository.increment_access_count(token).await?; + if updated == 0 { + return Err(ShareServiceError::NotFound(format!( + "Share with token {} not found or expired", + token + )) + .into()); } - - // Increment the access counter - let updated_share = share.increment_access_count(); - - // Save the changes - self.share_repository - .update_share(&updated_share) - .await - .map_err(|e| ShareServiceError::Repository(e.to_string()))?; - Ok(()) } } diff --git a/src/application/services/storage_usage_service.rs b/src/application/services/storage_usage_service.rs index 637419a2..f2fef771 100644 --- a/src/application/services/storage_usage_service.rs +++ b/src/application/services/storage_usage_service.rs @@ -1,4 +1,3 @@ -use crate::application::ports::auth_ports::UserStoragePort; use crate::application::ports::storage_ports::StorageUsagePort; use crate::common::errors::DomainError; use crate::infrastructure::repositories::pg::UserPgRepository; @@ -512,9 +511,9 @@ impl StorageUsagePort for StorageUsageService { user_id: Uuid, additional_bytes: u64, ) -> Result<(), DomainError> { - let user = self.user_repository.get_user_by_id(user_id).await?; - let quota = user.storage_quota_bytes(); - let used = user.storage_used_bytes(); + // Narrow 2-column read — the full user row carries the up-to-512 KiB + // avatar `image` column, paid on every upload quota check otherwise. + let (used, quota) = self.user_repository.get_storage_usage(user_id).await?; // Quota of 0 means unlimited if quota <= 0 { @@ -548,8 +547,9 @@ impl StorageUsagePort for StorageUsageService { } async fn get_user_storage_info(&self, user_id: Uuid) -> Result<(i64, i64), DomainError> { - let user = self.user_repository.get_user_by_id(user_id).await?; - Ok((user.storage_used_bytes(), user.storage_quota_bytes())) + // Narrow 2-column read (avatar-free) — runs on every folder PROPFIND + // that reports quota. See benches/QUOTA-PATH.md. + Ok(self.user_repository.get_storage_usage(user_id).await?) } async fn add_drive_storage_usage_delta( diff --git a/src/application/services/trash_service.rs b/src/application/services/trash_service.rs index 045b8ea3..5892f962 100644 --- a/src/application/services/trash_service.rs +++ b/src/application/services/trash_service.rs @@ -14,7 +14,7 @@ use crate::application::dtos::trash_dto::{ }; use crate::application::ports::authorization_ports::AuthorizationEngine; use crate::application::ports::file_lifecycle::FileLifecycleHook; -use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; +use crate::application::ports::storage_ports::FileWritePort; use crate::application::ports::trash_ports::TrashUseCase; use crate::common::errors::{DomainError, ErrorKind, Result}; use crate::domain::entities::file::File; @@ -24,7 +24,6 @@ use crate::domain::repositories::folder_repository::FolderRepository; use crate::domain::repositories::trash_repository::TrashRepository; use crate::domain::services::authorization::ResourceKind; use crate::domain::services::authorization::{Permission, Resource, Subject}; -use crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlobReadRepository; use crate::infrastructure::repositories::pg::file_blob_write_repository::FileBlobWriteRepository; use crate::infrastructure::repositories::pg::folder_db_repository::FolderDbRepository; use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository; @@ -49,9 +48,6 @@ pub struct TrashService { /// Repository for trash-specific operations like listing and retrieving trashed items trash_repository: Arc, - /// Port for file read operations (get file metadata) - file_read_port: Arc, - /// Port for file write operations (trash, restore, delete) file_write_port: Arc, @@ -75,19 +71,14 @@ pub struct TrashService { /// so trash listings filter by drive membership instead of the legacy /// per-user scope. drive_repo: Arc, - - /// Number of days items should be kept in trash before automatic cleanup - retention_days: u32, } impl TrashService { #[allow(clippy::too_many_arguments)] pub fn new( trash_repository: Arc, - file_read_port: Arc, file_write_port: Arc, folder_storage_port: Arc, - retention_days: u32, dedup_service: Arc, content_cache: Option>, authz: Arc, @@ -95,7 +86,6 @@ impl TrashService { ) -> Self { Self { trash_repository, - file_read_port, file_write_port, folder_storage_port, dedup_service, @@ -103,7 +93,6 @@ impl TrashService { content_cache, authz, drive_repo, - retention_days, } } @@ -177,23 +166,17 @@ impl TrashUseCase for TrashService { // Note: We now verify file/folder ownership BEFORE moving to trash. // This prevents users from trashing items they do not own (IDOR). - // Parse UUIDs with detailed error handling + // Parse UUIDs with detailed error handling. The parsed value is + // re-derived per branch below; this early check preserves the 400 + // (validation) error shape for malformed ids. debug!("Validating item UUID: {}", item_id); - let item_uuid = match Uuid::parse_str(item_id) { - Ok(uuid) => { - debug!("Valid item UUID: {}", uuid); - uuid - } - Err(e) => { - error!("Invalid item UUID: {} - Error: {}", item_id, e); - return Err(DomainError::validation_error(format!( - "Invalid item ID: {}", - e - ))); - } - }; - - let user_uuid = user_id; + if let Err(e) = Uuid::parse_str(item_id) { + error!("Invalid item UUID: {} - Error: {}", item_id, e); + return Err(DomainError::validation_error(format!( + "Invalid item ID: {}", + e + ))); + } match item_type { "file" => { @@ -209,59 +192,13 @@ impl TrashUseCase for TrashService { ) .await?; - // Authz already passed — use the non-owner-scoped read so that - // grantees with Delete permission can trash files they don't own. - // The file's user_id in storage.files is unchanged, so the item - // will appear in the original owner's trash view. - let file = match self.file_read_port.get_file(item_id).await { - Ok(file) => { - debug!("File found: {} ({})", file.name(), item_id); - file - } - Err(e) => { - error!("Error getting file: {} - {}", item_id, e); - return Err(DomainError::new( - ErrorKind::NotFound, - "File", - format!("Error retrieving file {}: {}", item_id, e), - )); - } - }; - - let original_path = file.storage_path().to_string(); - debug!("Original file path: {}", original_path); - - debug!("Creating TrashedItem object for the file"); - let trashed_item = TrashedItem::new( - item_uuid, - user_uuid, - TrashedItemType::File, - file.name().to_string(), - original_path, - self.retention_days, - ); - debug!( - "TrashedItem created successfully: {} -> {}", - file.name(), - trashed_item.id() - ); - - // First add to trash index to register the item - info!("Adding file {} to trash index", item_id); - match self.trash_repository.add_to_trash(&trashed_item).await { - Ok(_) => { - debug!("File added to trash index successfully"); - } - Err(e) => { - error!("Error adding file to trash index: {}", e); - return Err(DomainError::internal_error( - "TrashRepository", - format!("Failed to add file to trash: {}", e), - )); - } - }; - - // Then physically move the file to trash. + // Soft-delete model: the is_trashed flag on the row IS the + // trash membership — there is no separate trash index to + // register into (`TrashRepository::add_to_trash` is a + // documented no-op). The previous shape still fetched the + // full file entity and built a `TrashedItem` only to feed + // that no-op: one wasted SELECT per trash operation. + // // §14: caller_id stamps `updated_by` on the trashed row. info!("Physically moving file to trash: {}", item_id); match self.file_write_port.move_to_trash(item_id, user_id).await { @@ -293,43 +230,10 @@ impl TrashUseCase for TrashService { ) .await?; - let folder = self - .folder_storage_port - .get_folder(item_id) - .await - .map_err(|e| { - DomainError::new( - ErrorKind::NotFound, - "Folder", - format!("Error retrieving folder {}: {}", item_id, e), - ) - })?; - - let original_path = folder.storage_path().to_string(); - - let trashed_item = TrashedItem::new( - item_uuid, - user_uuid, - TrashedItemType::Folder, - folder.name().to_string(), - original_path, - self.retention_days, - ); - - // First add to trash index to register the item - debug!("Adding folder {} to trash repository", item_id); - match self.trash_repository.add_to_trash(&trashed_item).await { - Ok(_) => debug!("Successfully added folder to trash repository"), - Err(e) => { - error!("Failed to add folder to trash repository: {}", e); - return Err(DomainError::internal_error( - "TrashRepository", - format!("Failed to add folder to trash: {}", e), - )); - } - }; - - // Then physically move the folder to trash. + // Soft-delete model — same as the file branch above: the + // cascade UPDATE below is the whole operation; no folder + // fetch or trash-index write needed. + // // §14: caller_id stamps `updated_by` on every cascade-trashed row. self.folder_storage_port .move_to_trash(item_id, user_id) diff --git a/src/common/di.rs b/src/common/di.rs index e87969fc..75166a86 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -836,10 +836,8 @@ impl AppServiceFactory { let service = Arc::new( TrashService::new( trash_repo.clone(), - repos.file_read_repository.clone(), repos.file_write_repository.clone(), repos.folder_repository.clone(), - self.config.storage.trash_retention_days, core.dedup_service.clone(), Some(core.file_content_cache.clone()), authz.clone(), diff --git a/src/common/mime_detect.rs b/src/common/mime_detect.rs index 1022ea03..55d59867 100644 --- a/src/common/mime_detect.rs +++ b/src/common/mime_detect.rs @@ -108,10 +108,117 @@ pub async fn refine_content_type_from_file( } } +/// Whether a MIME type identifies content that is already compressed, so +/// running Deflate over it burns CPU for ~0 % size gain. +/// +/// Used by the ZIP export paths (`ZipService`, `BatchOperations`) to pick +/// `Compression::Stored` per entry instead of deflating JPEG/MP4/… bytes. +/// The set mirrors the HTTP `CompressionLayer` exclusion list in `main.rs` +/// (keep the two in sync), minus entries that are containers of possibly +/// incompressible data rather than compressed formats themselves +/// (`application/x-tar`, `application/octet-stream`) — those stay on Deflate +/// so unknown-but-compressible content is never stored uncompressed. +pub fn is_precompressed_mime(mime: &str) -> bool { + // Strip any parameters ("; charset=…") and normalize case. + let essence = mime.split(';').next().unwrap_or(mime).trim(); + + // Compressed families: every common video/audio codec container. + if essence.starts_with("video/") || essence.starts_with("audio/") { + return true; + } + // Zip-based document bundles (docx/xlsx/pptx, odt/ods/odp, …). + if essence.starts_with("application/vnd.openxmlformats-officedocument") + || essence.starts_with("application/vnd.oasis.opendocument") + { + return true; + } + + matches!( + essence, + // Raster images with built-in compression (SVG intentionally absent). + "image/jpeg" + | "image/png" + | "image/gif" + | "image/webp" + | "image/avif" + | "image/heic" + | "image/heif" + | "image/jp2" + // Already-compressed web fonts; ttf/otf left compressible. + | "font/woff" + | "font/woff2" + | "application/font-woff" + // Archives & compressed containers. + | "application/zip" + | "application/gzip" + | "application/x-gzip" + | "application/x-7z-compressed" + | "application/x-rar-compressed" + | "application/x-bzip2" + | "application/zstd" + | "application/x-xz" + | "application/epub+zip" + | "application/java-archive" + | "application/vnd.android.package-archive" + // PDF: internal streams are usually already deflated. + | "application/pdf" + ) +} + +/// ZIP entry compression for a file of the given MIME type: `Stored` for +/// already-compressed content, `Deflate` otherwise. Shared by every ZIP +/// export path (`ZipService`, `BatchOperations`). +pub fn zip_entry_compression(mime: &str) -> async_zip::Compression { + if is_precompressed_mime(mime) { + async_zip::Compression::Stored + } else { + async_zip::Compression::Deflate + } +} + #[cfg(test)] mod tests { use super::*; + // ── is_precompressed_mime ─────────────────────────────────── + + #[test] + fn media_and_archives_are_precompressed() { + for mime in [ + "image/jpeg", + "image/webp", + "video/mp4", + "video/quicktime", + "audio/mpeg", + "application/zip", + "application/pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "font/woff2", + ] { + assert!(is_precompressed_mime(mime), "{mime} should be Stored"); + } + } + + #[test] + fn compressible_types_keep_deflate() { + for mime in [ + "text/plain", + "text/html", + "application/json", + "image/svg+xml", + "application/x-tar", + "application/octet-stream", + "", + ] { + assert!(!is_precompressed_mime(mime), "{mime} should stay Deflate"); + } + } + + #[test] + fn mime_parameters_are_ignored() { + assert!(is_precompressed_mime("image/jpeg; charset=binary")); + } + // ── refine_content_type (sync) ────────────────────────────── #[test] diff --git a/src/domain/services/path_service.rs b/src/domain/services/path_service.rs index e1ebbe71..aa2b0291 100644 --- a/src/domain/services/path_service.rs +++ b/src/domain/services/path_service.rs @@ -5,7 +5,7 @@ //! infrastructure/services/path_service.rs because it has file system dependencies. use std::path::PathBuf; -use unicode_normalization::UnicodeNormalization; +use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick}; /// NFC-normalize a single file or folder name component. /// @@ -25,7 +25,18 @@ use unicode_normalization::UnicodeNormalization; /// (`migrate-nfc-filenames`) cleans up rows that pre-date this rule. /// /// Pure function — no I/O, allocates one `String`. +/// +/// Fast path: `is_nfc_quick` is a per-char table lookup that answers +/// `Yes` for virtually every name already in NFC — which is every name +/// loaded back from PostgreSQL (the DB invariant above) and every +/// ASCII name. That skips the full decompose/recompose state machine +/// this function otherwise runs once per row on every listing +/// (PROPFIND, folder listing, photos timeline). `Maybe`/`No` fall +/// through to the full pipeline. pub fn normalize_storage_name(name: &str) -> String { + if is_nfc_quick(name.chars()) == IsNormalized::Yes { + return name.to_string(); + } name.nfc().collect() } diff --git a/src/infrastructure/repositories/pg/drive_pg_repository.rs b/src/infrastructure/repositories/pg/drive_pg_repository.rs index c52171f6..89e9b494 100644 --- a/src/infrastructure/repositories/pg/drive_pg_repository.rs +++ b/src/infrastructure/repositories/pg/drive_pg_repository.rs @@ -10,7 +10,9 @@ //! schema and `docs/plan/drive.md` §3 / §15 for the locked design. use std::sync::Arc; +use std::time::Duration; +use moka::future::Cache; use sqlx::{PgPool, Row, types::Uuid}; use crate::domain::entities::drive::{Drive, DriveKind}; @@ -18,13 +20,38 @@ use crate::domain::repositories::drive_repository::{ DriveRepository, DriveRepositoryError, DriveWithRootName, }; +/// `default_drive_cache` TTL. The default-drive → root-folder binding is +/// nearly immutable (changes only on provisioning / drive deletion / +/// policy edits — all of which invalidate explicitly below), yet it is +/// re-resolved on EVERY NextCloud request (basic-auth chroot), every +/// native `/webdav` request (Mode-B scope resolution) and every WOPI +/// call. 30 s mirrors `drive_role_cache` in `pg_acl_engine.rs` and bounds +/// the one non-invalidated staleness source: a root-folder *rename*, +/// which doesn't pass through this repository. Measured in +/// `benches/CHROOT-CACHE.md`. +const DEFAULT_DRIVE_CACHE_TTL: Duration = Duration::from_secs(30); + +/// One entry per active user; entries are small (a `Drive` + a name). +const DEFAULT_DRIVE_CACHE_CAPACITY: u64 = 100_000; + pub struct DrivePgRepository { pool: Arc, + /// user_id → default drive (+ root folder name). See + /// [`DEFAULT_DRIVE_CACHE_TTL`]. Only `Ok` results are cached, so the + /// provisioning idempotency check (`NotFound` → create) always sees + /// the live table. + default_drive_cache: Cache, } impl DrivePgRepository { pub fn new(pool: Arc) -> Self { - Self { pool } + Self { + pool, + default_drive_cache: Cache::builder() + .max_capacity(DEFAULT_DRIVE_CACHE_CAPACITY) + .time_to_live(DEFAULT_DRIVE_CACHE_TTL) + .build(), + } } fn map_sqlx_err(context: &'static str, e: sqlx::Error) -> DriveRepositoryError { @@ -209,6 +236,10 @@ impl DriveRepository for DrivePgRepository { .await .map_err(|e| Self::map_sqlx_err("create_personal_drive_atomic.commit", e))?; + // Drop any cached default-drive resolution for this user (a stale + // NotFound is never cached, but be explicit about the write path). + self.default_drive_cache.invalidate(&owner_id).await; + Self::row_to_drive_with_name(&row) } @@ -394,6 +425,10 @@ impl DriveRepository for DrivePgRepository { tx.commit() .await .map_err(|e| Self::map_sqlx_err("delete_atomic.commit", e))?; + // We only have the drive id here; the cache is keyed by user. + // Deletion is rare — clearing the whole cache is the simple, + // always-correct move (repopulates at one query per active user). + self.default_drive_cache.invalidate_all(); Ok(()) } @@ -448,6 +483,10 @@ impl DriveRepository for DrivePgRepository { &self, user_id: Uuid, ) -> Result { + if let Some(cached) = self.default_drive_cache.get(&user_id).await { + return Ok(cached); + } + let row = sqlx::query( r#" SELECT d.id, d.kind, d.default_for_user, d.root_folder_id, @@ -465,7 +504,9 @@ impl DriveRepository for DrivePgRepository { .map_err(|e| Self::map_sqlx_err("find_default_for_user", e))? .ok_or_else(|| DriveRepositoryError::NotFound(user_id.to_string()))?; - Self::row_to_drive_with_name(&row) + let dwr = Self::row_to_drive_with_name(&row)?; + self.default_drive_cache.insert(user_id, dwr.clone()).await; + Ok(dwr) } async fn list_readable_by( @@ -675,6 +716,10 @@ impl DriveRepository for DrivePgRepository { let raw = row .ok_or_else(|| DriveRepositoryError::NotFound(drive_id.to_string()))? .0; + // Policy edits must not serve a stale `policies` bag from the + // default-drive cache (keyed by user, and we only have the drive + // id) — clear it; policy edits are admin-rare. + self.default_drive_cache.invalidate_all(); Ok(crate::domain::entities::drive::DrivePolicies::from_value( &raw, )) diff --git a/src/infrastructure/repositories/pg/face_pg_repository.rs b/src/infrastructure/repositories/pg/face_pg_repository.rs index a417e382..bfec803c 100644 --- a/src/infrastructure/repositories/pg/face_pg_repository.rs +++ b/src/infrastructure/repositories/pg/face_pg_repository.rs @@ -186,6 +186,60 @@ impl FaceRepository for FacePgRepository { Ok(rows.into_iter().map(row_to_face).collect()) } + async fn person_face_stats(&self, user_id: Uuid) -> Result, DomainError> { + // Grouped COUNT — the People tab only needs per-person counts, so + // this replaces a full faces_for_user scan that shipped a 2 KiB + // embedding BYTEA per row (benches/PEOPLE-LIST.md). + let rows: Vec<(Uuid, i64)> = sqlx::query_as( + "SELECT person_id, COUNT(*) FROM faces.faces + WHERE user_id = $1 AND person_id IS NOT NULL + GROUP BY person_id", + ) + .bind(user_id) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| db_err("person_face_stats", e))?; + Ok(rows) + } + + async fn file_ids_for_faces( + &self, + user_id: Uuid, + face_ids: &[Uuid], + ) -> Result, DomainError> { + if face_ids.is_empty() { + return Ok(std::collections::HashMap::new()); + } + let rows: Vec<(Uuid, Uuid)> = sqlx::query_as( + "SELECT id, file_id FROM faces.faces WHERE user_id = $1 AND id = ANY($2)", + ) + .bind(user_id) + .bind(face_ids) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| db_err("file_ids_for_faces", e))?; + Ok(rows.into_iter().collect()) + } + + async fn reassign_person_faces( + &self, + user_id: Uuid, + from: Uuid, + into: Uuid, + ) -> Result { + let result = sqlx::query( + "UPDATE faces.faces SET person_id = $3 + WHERE user_id = $1 AND person_id = $2", + ) + .bind(user_id) + .bind(from) + .bind(into) + .execute(self.pool.as_ref()) + .await + .map_err(|e| db_err("reassign_person_faces", e))?; + Ok(result.rows_affected()) + } + async fn assign_person( &self, face_id: Uuid, diff --git a/src/infrastructure/repositories/pg/file_blob_read_repository.rs b/src/infrastructure/repositories/pg/file_blob_read_repository.rs index 2de4393e..e631aa7c 100644 --- a/src/infrastructure/repositories/pg/file_blob_read_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_read_repository.rs @@ -366,6 +366,30 @@ impl FileBlobReadRepository { .ok_or_else(|| DomainError::not_found("File", file_id)) } + /// Batched variant of [`Self::get_file_drive_id`]: one `= ANY($1)` + /// round-trip for a whole result page. Missing / unknown ids are simply + /// absent from the output (the single-id variant maps them to + /// `NotFound`). Used by `PgAclEngine::check_files_read_batch` — the + /// per-hit loop cost up to 200 sequential point SELECTs per content + /// search (benches/SEARCH-REBAC.md). + pub async fn get_file_drive_ids( + &self, + file_ids: &[uuid::Uuid], + ) -> Result, DomainError> { + if file_ids.is_empty() { + return Ok(Vec::new()); + } + sqlx::query_as::<_, (uuid::Uuid, uuid::Uuid)>( + "SELECT id, drive_id FROM storage.files WHERE id = ANY($1)", + ) + .bind(file_ids) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("FileBlobRead", format!("drive_id batch lookup: {e}")) + }) + } + /// Creates a stub instance for testing — never hits PG. /// Available in both standard unit-test (`cfg(test)`) and integration /// (`cfg(integration_tests)`) builds; `PgAclEngine::new_stub` chains @@ -494,7 +518,24 @@ impl FileBlobReadRepository { before: Option, limit: i64, ) -> Result<(Vec, Vec, Vec<(Option, Option)>), DomainError> { - let rows: Vec = sqlx::query_as( + // Sargable keyset cursor: compare the RAW `media_sort_date` column + // against a timestamptz bind so the planner can use the cursor as + // an index boundary condition on `idx_files_media_timeline_by_drive`. + // The old shape wrapped the column in `EXTRACT(EPOCH …)::bigint` + // (plus an `IS NULL OR` disjunction), which degraded the cursor to + // a per-row Filter: page k re-read and discarded all k·limit rows + // already scrolled past (benches/PHOTOS-CURSOR.md). Since `before` + // is whole seconds, `media_sort_date < to_timestamp(before)` admits + // exactly the same rows as the old truncated comparison. The + // predicate is emitted only when a cursor exists — a bound + // disjunction would block the index condition under generic plans. + let cursor_ts = before.and_then(|s| chrono::DateTime::from_timestamp(s, 0)); + let cursor_pred = if cursor_ts.is_some() { + "AND fi.media_sort_date < $2" + } else { + "AND $2::timestamptz IS NULL" + }; + let sql = format!( r#" SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, fi.size, fi.mime_type, @@ -524,18 +565,18 @@ impl FileBlobReadRepository { ) AND NOT fi.is_trashed AND (fi.mime_type LIKE 'image/%' OR fi.mime_type LIKE 'video/%') - AND ($2::bigint IS NULL - OR EXTRACT(EPOCH FROM fi.media_sort_date)::bigint < $2::bigint) + {cursor_pred} ORDER BY fi.media_sort_date DESC LIMIT $3 "#, - ) - .bind(caller_id) - .bind(before) - .bind(limit) - .fetch_all(self.pool.as_ref()) - .await - .map_err(|e| DomainError::internal_error("FileBlobRead", format!("list_media: {e}")))?; + ); + let rows: Vec = sqlx::query_as(&sql) + .bind(caller_id) + .bind(cursor_ts) + .bind(limit) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("FileBlobRead", format!("list_media: {e}")))?; let mut files = Vec::with_capacity(rows.len()); let mut sort_dates = Vec::with_capacity(rows.len()); @@ -768,62 +809,57 @@ impl FileReadPort for FileBlobReadRepository { self.resolve_blob_hash(file_id).await } - /// Paginated file listing — fetches only `limit` rows starting at `offset`. + /// Keyset-paginated file listing in name order — fetches only `limit` + /// rows after `after_name` (exclusive). /// - /// Uses a single SQL query with `LIMIT/OFFSET` to avoid loading the full - /// folder contents into memory. Ideal for streaming WebDAV PROPFIND. + /// Names are unique per folder, so `name > $after` is a total cursor. + /// Served by `idx_files_folder_name (folder_id, name) WHERE NOT + /// is_trashed` as a pure index-range read: O(page) per page with no + /// sort, where the old `LIMIT/OFFSET` shape re-scanned and re-sorted + /// the entire folder for every page (benches/PROPFIND-PAGING.md). The + /// cursor predicate is emitted only when a cursor exists — a + /// `$2 IS NULL OR name > $2` disjunction would block the index + /// condition under the extended protocol's generic plans. #[allow(clippy::type_complexity)] async fn list_files_batch( &self, folder_id: Option<&str>, - offset: i64, + after_name: Option<&str>, limit: i64, ) -> Result, DomainError> { - let rows: Vec = if let Some(fid) = folder_id { - sqlx::query_as( - r#" - SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, - fi.size, fi.mime_type, - EXTRACT(EPOCH FROM fi.created_at)::bigint, - EXTRACT(EPOCH FROM fi.updated_at)::bigint, - fi.blob_hash, - - fi.created_by, fi.updated_by - FROM storage.files fi - LEFT JOIN storage.folders fo ON fo.id = fi.folder_id - WHERE fi.folder_id = $1::uuid AND NOT fi.is_trashed - ORDER BY fi.name - LIMIT $2 OFFSET $3 - "#, - ) - .bind(fid) - .bind(limit) - .bind(offset) - .fetch_all(self.pool.as_ref()) - .await + let folder_pred = if folder_id.is_some() { + "fi.folder_id = $1::uuid" } else { - sqlx::query_as( - r#" - SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, - fi.size, fi.mime_type, - EXTRACT(EPOCH FROM fi.created_at)::bigint, - EXTRACT(EPOCH FROM fi.updated_at)::bigint, - fi.blob_hash, + "fi.folder_id IS NULL AND $1::uuid IS NULL" + }; + let cursor_pred = if after_name.is_some() { + "AND fi.name > $3" + } else { + "AND $3::text IS NULL" + }; + let sql = format!( + r#" + SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path, + fi.size, fi.mime_type, + EXTRACT(EPOCH FROM fi.created_at)::bigint, + EXTRACT(EPOCH FROM fi.updated_at)::bigint, + fi.blob_hash, - fi.created_by, fi.updated_by - FROM storage.files fi - LEFT JOIN storage.folders fo ON fo.id = fi.folder_id - WHERE fi.folder_id IS NULL AND NOT fi.is_trashed - ORDER BY fi.name - LIMIT $1 OFFSET $2 - "#, - ) + fi.created_by, fi.updated_by + FROM storage.files fi + LEFT JOIN storage.folders fo ON fo.id = fi.folder_id + WHERE {folder_pred} AND NOT fi.is_trashed {cursor_pred} + ORDER BY fi.name + LIMIT $2 + "#, + ); + let rows: Vec = sqlx::query_as(&sql) + .bind(folder_id) .bind(limit) - .bind(offset) + .bind(after_name) .fetch_all(self.pool.as_ref()) .await - } - .map_err(|e| DomainError::internal_error("FileBlobRead", format!("list_batch: {e}")))?; + .map_err(|e| DomainError::internal_error("FileBlobRead", format!("list_batch: {e}")))?; rows.into_iter() .map( diff --git a/src/infrastructure/repositories/pg/share_pg_repository.rs b/src/infrastructure/repositories/pg/share_pg_repository.rs index 8660324e..8850a61f 100644 --- a/src/infrastructure/repositories/pg/share_pg_repository.rs +++ b/src/infrastructure/repositories/pg/share_pg_repository.rs @@ -122,6 +122,35 @@ impl ShareStoragePort for SharePgRepository { Self::row_to_entity(&row) } + async fn increment_access_count(&self, token: &str) -> Result { + // One atomic statement — the relative bump can't lose concurrent + // increments and never rewrites unrelated columns (the legacy + // read-modify-write wrote back item_name/password_hash wholesale, + // silently clobbering concurrent owner edits). The expiry guard + // mirrors find_share_by_token's MIN(expires_at) subquery: NULL = + // never expires. + let result = sqlx::query( + r#" + UPDATE storage.shares s + SET access_count = s.access_count + 1 + WHERE s.token = $1 + AND COALESCE( + (SELECT MIN(ag.expires_at) + FROM storage.role_grants ag + WHERE ag.subject_type = 'token' AND ag.subject_id = s.id) > NOW(), + TRUE) + "#, + ) + .bind(token) + .execute(&*self.db_pool) + .await + .map_err(|e| { + tracing::error!("Database error incrementing share access count: {}", e); + DomainError::internal_error("Share", format!("Failed to register access: {e}")) + })?; + Ok(result.rows_affected()) + } + async fn find_share_by_token(&self, token: &str) -> Result { let row = sqlx::query( r#" diff --git a/src/infrastructure/repositories/pg/user_pg_repository.rs b/src/infrastructure/repositories/pg/user_pg_repository.rs index 5fa36c5c..7847fc65 100644 --- a/src/infrastructure/repositories/pg/user_pg_repository.rs +++ b/src/infrastructure/repositories/pg/user_pg_repository.rs @@ -85,6 +85,33 @@ impl UserPgRepository { }) } + /// Fetch only `(storage_used_bytes, storage_quota_bytes)`. Not part of + /// the `UserRepository` trait — called from `StorageUsageService`. + /// + /// Same rationale as [`Self::get_user_flags`]: the full-row SELECT drags + /// `image` (a data URI of up to 512 KiB), `password_hash`, + /// `ui_preferences`, … across the wire, and the quota path runs on every + /// folder PROPFIND and every upload quota check just to read two i64s. + /// Measured in `benches/QUOTA-PATH.md`. + pub async fn get_storage_usage(&self, id: Uuid) -> UserRepositoryResult<(i64, i64)> { + let row = sqlx::query( + r#" + SELECT storage_used_bytes, storage_quota_bytes + FROM auth.users + WHERE id = $1 + "#, + ) + .bind(id) + .fetch_one(&*self.pool) + .await + .map_err(Self::map_sqlx_error)?; + + Ok(( + row.get("storage_used_bytes"), + row.get("storage_quota_bytes"), + )) + } + /// Updates a user's profile image (URL or data URI). Not part of the /// `UserRepository` trait — called directly from `AuthApplicationService`. pub async fn update_image( diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index f9e6c263..38e429d5 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -240,6 +240,16 @@ impl Drop for IngestGuard { /// in the [`BlobStorageBackend`], and maintains a manifest in PostgreSQL /// mapping file_hash → \[chunk_hashes\]. BLAKE3 hashing, ref-counting /// and the PostgreSQL dedup index all live here. +/// Immutable chunk map of one CDC blob (`storage.chunk_manifests` row, +/// minus the mutable `ref_count`). Content-addressed: for a given +/// `file_hash` the chunk list and total size never change, which is what +/// makes [`DedupService::manifest_cached`] safe. +pub struct ChunkManifest { + pub chunk_hashes: Vec, + pub chunk_sizes: Vec, + pub total_size: i64, +} + pub struct DedupService { /// Pluggable blob storage backend (local FS, S3, …). backend: Arc, @@ -251,6 +261,13 @@ pub struct DedupService { maintenance_pool: Arc, /// Single lifecycle dispatcher — fired on blob created / deleted. blob_lifecycle: Option>, + /// `file_hash → ChunkManifest` for the read path — every stream / range + /// / full read of a CDC blob used to pay one manifest query first, even + /// for the media the gallery re-reads constantly. Positive-only (a + /// legacy blob gaining a manifest via background rechunking must be + /// seen immediately), weight-bounded (a manifest is ~72 B per chunk), + /// short TTL so GC'd manifests age out fast (benches/MANIFEST-CACHE.md). + manifest_cache: moka::future::Cache>, } impl DedupService { @@ -269,9 +286,22 @@ impl DedupService { pool, maintenance_pool, blob_lifecycle: None, + manifest_cache: Self::build_manifest_cache(), } } + /// See the `manifest_cache` field docs. Weight ≈ real heap bytes of one + /// entry; 32 MiB cap ≈ tens of thousands of typical (sub-1 GB) files. + fn build_manifest_cache() -> moka::future::Cache> { + moka::future::Cache::builder() + .weigher(|key: &String, value: &Arc| { + (key.len() + value.chunk_hashes.len() * 80 + 64) as u32 + }) + .max_capacity(32 * 1024 * 1024) + .time_to_live(std::time::Duration::from_secs(60)) + .build() + } + /// Registers the blob lifecycle dispatcher (thumbnail cleanup, …). pub fn with_blob_lifecycle(mut self, lifecycle: Arc) -> Self { self.blob_lifecycle = Some(lifecycle); @@ -311,6 +341,7 @@ impl DedupService { pool: stub_pool.clone(), maintenance_pool: stub_pool, blob_lifecycle: None, + manifest_cache: Self::build_manifest_cache(), } } @@ -1385,6 +1416,10 @@ impl DedupService { .await .map_err(|e| DomainError::internal_error("Dedup", format!("Commit: {}", e)))?; + // Post-commit so a concurrent read can't re-cache the manifest + // between invalidation and the delete becoming visible. + self.manifest_cache.invalidate(file_hash).await; + // File content is gone — drop its blob-keyed thumbnails now. self.fire_blob_hooks(file_hash); @@ -1606,27 +1641,49 @@ impl DedupService { Box::pin(chunk_stream) } + /// Cached manifest fetch for the read path (see the `manifest_cache` + /// field docs). `None` = legacy whole-file blob — never cached, so a + /// background rechunk that creates a manifest is honoured immediately. + async fn manifest_cached(&self, hash: &str) -> Result>, DomainError> { + if let Some(m) = self.manifest_cache.get(hash).await { + return Ok(Some(m)); + } + let row = sqlx::query_as::<_, (Vec, Vec, i64)>( + "SELECT chunk_hashes, chunk_sizes, total_size + FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(hash) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?; + match row { + Some((chunk_hashes, chunk_sizes, total_size)) => { + let m = Arc::new(ChunkManifest { + chunk_hashes, + chunk_sizes, + total_size, + }); + self.manifest_cache + .insert(hash.to_string(), m.clone()) + .await; + Ok(Some(m)) + } + None => Ok(None), + } + } + /// Stream blob content — CDC-aware with legacy fallback. /// - /// For CDC files: looks up the manifest, then streams chunks in order, - /// concatenating them into a single byte stream. + /// For CDC files: looks up the manifest (RAM-cached), then streams + /// chunks in order, concatenating them into a single byte stream. /// For legacy blobs: delegates directly to the backend. pub async fn read_blob_stream( &self, hash: &str, ) -> Result> + Send>>, DomainError> { - // Check manifest - let manifest = sqlx::query_scalar::<_, Vec>( - "SELECT chunk_hashes FROM storage.chunk_manifests WHERE file_hash = $1", - ) - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?; - - match manifest { - Some(chunk_hashes) => Ok(self.stream_chunks(chunk_hashes)), + match self.manifest_cached(hash).await? { + Some(m) => Ok(self.stream_chunks(m.chunk_hashes.clone())), // Legacy whole-file blob None => self.backend.get_blob_stream(hash).await, } @@ -1644,18 +1701,11 @@ impl DedupService { /// `blob_size` + `read_blob_stream`) doubled the manifest round-trips on /// every full-blob read (e.g. 2N queries for an N-image gallery cold load). pub async fn read_blob_bytes(&self, hash: &str) -> Result { - let manifest = sqlx::query_as::<_, (Vec, i64)>( - "SELECT chunk_hashes, total_size FROM storage.chunk_manifests WHERE file_hash = $1", - ) - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?; - - let (mut stream, expected_size) = match manifest { - Some((chunk_hashes, total_size)) => { - (self.stream_chunks(chunk_hashes), total_size.max(0) as usize) - } + let (mut stream, expected_size) = match self.manifest_cached(hash).await? { + Some(m) => ( + self.stream_chunks(m.chunk_hashes.clone()), + m.total_size.max(0) as usize, + ), None => { // Legacy whole-file blob: size + stream straight from the backend. let size = self.backend.blob_size(hash).await? as usize; @@ -1685,17 +1735,9 @@ impl DedupService { end: Option, ) -> Result> + Send>>, DomainError> { - // Check manifest - let manifest = sqlx::query_as::<_, (Vec, Vec, i64)>( - "SELECT chunk_hashes, chunk_sizes, total_size - FROM storage.chunk_manifests WHERE file_hash = $1", - ) - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?; - - if let Some((chunk_hashes, chunk_sizes, total_size)) = manifest { + if let Some(m) = self.manifest_cached(hash).await? { + let (chunk_hashes, chunk_sizes, total_size) = + (&m.chunk_hashes, &m.chunk_sizes, m.total_size); let end = end.unwrap_or(total_size as u64); // Calculate which chunks overlap [start, end) @@ -1749,17 +1791,9 @@ impl DedupService { /// Get blob size — manifest-aware with legacy fallback. pub async fn blob_size(&self, hash: &str) -> Result { - // Check manifest first (O(1) from PG) - let manifest_size = sqlx::query_scalar::<_, i64>( - "SELECT total_size FROM storage.chunk_manifests WHERE file_hash = $1", - ) - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {}", e)))?; - - if let Some(size) = manifest_size { - return Ok(size as u64); + // Check manifest first (RAM cache, else one O(1) PG row) + if let Some(m) = self.manifest_cached(hash).await? { + return Ok(m.total_size as u64); } // Legacy: delegate to backend @@ -2048,6 +2082,7 @@ impl DedupService { } for (file_hash, chunk_hashes, size) in &batch { + self.manifest_cache.invalidate(file_hash).await; // Decrement chunk ref_counts. GREATEST(.., 0) guards against the // single-chunk file case where the PG file-delete trigger already // decremented blobs.ref_count (because file_hash == chunk_hash); diff --git a/src/infrastructure/services/pg_acl_engine.rs b/src/infrastructure/services/pg_acl_engine.rs index 3f79aaf3..64973a5c 100644 --- a/src/infrastructure/services/pg_acl_engine.rs +++ b/src/infrastructure/services/pg_acl_engine.rs @@ -1112,6 +1112,78 @@ impl AuthorizationEngine for PgAclEngine { result } + /// Batched Read check over a page of file ids (see the trait docs). + /// + /// Decision-equivalent to looping `check`: (1) resolve every file's + /// drive in one `= ANY($1)` query (same rows as N × + /// `get_file_drive_id`; absent ids decide `false` exactly like the + /// per-file `NotFound` path), (2) evaluate the drive-role floor once + /// per distinct drive through the same `drive_role_cache`, (3) send + /// only the drive-floor misses through the full per-file cascade — + /// preserving per-file grant resolution. `Read` is never gated by the + /// read-only drive freeze, so skipping that branch changes nothing. + async fn check_files_read_batch( + &self, + subject: Subject, + file_ids: &[Uuid], + ) -> Result, DomainError> { + use std::collections::{HashMap, HashSet}; + let start = std::time::Instant::now(); + let counters = QueryCounters::default(); + + counters.sql_queries.fetch_add(1, Ordering::Relaxed); + let pairs = self.file_repo.get_file_drive_ids(file_ids).await?; + + // Prime the resource→drive cache — later single checks on these + // files (download, share) skip their point lookup too. + for (file_id, drive_id) in &pairs { + self.owner_cache + .insert(Resource::File(*file_id), *drive_id) + .await; + } + + let mut drive_readable: HashMap = HashMap::new(); + for (_, drive_id) in &pairs { + if !drive_readable.contains_key(drive_id) { + let ok = self + .caller_role_on_drive_cached(subject, *drive_id, &counters) + .await? + .is_some_and(|role| role.expand().contains(&Permission::Read)); + drive_readable.insert(*drive_id, ok); + } + } + + let mut allowed: HashSet = HashSet::with_capacity(pairs.len()); + for (file_id, drive_id) in &pairs { + if drive_readable.get(drive_id).copied().unwrap_or(false) { + allowed.insert(*file_id); + } else if self + .check_inner( + subject, + Permission::Read, + Resource::File(*file_id), + &counters, + ) + .await? + { + // Per-file / folder-cascade grant inside a drive the caller + // has no role on — rare, but must keep resolving. + allowed.insert(*file_id); + } + } + + tracing::debug!( + target: "oxicloud::authz", + event = "authz.check_files_read_batch", + subject = %subject, + files = file_ids.len(), + allowed = allowed.len(), + duration_us = start.elapsed().as_micros() as u64, + sql_queries = counters.sql_queries.load(Ordering::Relaxed), + ); + Ok(allowed) + } + async fn list_incoming_grants(&self, subject: Subject) -> Result, DomainError> { let counters = QueryCounters::default(); let (subject_types, subject_ids) = self.subject_match_set(subject, &counters).await?; diff --git a/src/infrastructure/services/webdav_dead_property_store.rs b/src/infrastructure/services/webdav_dead_property_store.rs index 50ed1201..acb44669 100644 --- a/src/infrastructure/services/webdav_dead_property_store.rs +++ b/src/infrastructure/services/webdav_dead_property_store.rs @@ -34,6 +34,7 @@ //! it was not handled by the path-based store either, so this is a //! parity decision, not a regression. +use std::collections::HashMap; use std::sync::Arc; use sqlx::{PgPool, Row}; @@ -126,17 +127,23 @@ impl DeadPropertyStore { } /// Delete a specific dead property. No-op if not present. + /// + /// Filters on the concrete id column (`folder_id = $1` / `file_id = $1`) + /// rather than the old `IS NOT DISTINCT FROM` pair — PostgreSQL cannot + /// serve `IS NOT DISTINCT FROM` from a B-tree index, so every lookup + /// degraded to a sequential scan as the table grew. The `=` shape is + /// served by the partial unique indexes from migration 20260830000001. + /// (Same rationale for `get_all` / `get` / the batched readers below — + /// measured in `benches/DEAD-PROPS.md`.) pub async fn remove(&self, r: ResourceRef, name: &QualifiedName) -> Result<(), DomainError> { - let (folder_id, file_id) = split_ref(r); - sqlx::query( + let (column, id) = split_ref(r); + sqlx::query(&format!( "DELETE FROM storage.webdav_dead_properties - WHERE folder_id IS NOT DISTINCT FROM $1 - AND file_id IS NOT DISTINCT FROM $2 - AND namespace = $3 - AND local_name = $4", - ) - .bind(folder_id) - .bind(file_id) + WHERE {column} = $1 + AND namespace = $2 + AND local_name = $3", + )) + .bind(id) .bind(&name.namespace) .bind(&name.name) .execute(&*self.pool) @@ -150,28 +157,64 @@ impl DeadPropertyStore { &self, r: ResourceRef, ) -> Result)>, DomainError> { - let (folder_id, file_id) = split_ref(r); - let rows = sqlx::query( + let (column, id) = split_ref(r); + let rows = sqlx::query(&format!( "SELECT namespace, local_name, value FROM storage.webdav_dead_properties - WHERE folder_id IS NOT DISTINCT FROM $1 - AND file_id IS NOT DISTINCT FROM $2", - ) - .bind(folder_id) - .bind(file_id) + WHERE {column} = $1", + )) + .bind(id) .fetch_all(&*self.pool) .await .map_err(|e| DomainError::internal_error("DeadPropertyStore", format!("get_all: {e}")))?; - Ok(rows - .into_iter() - .map(|r| { - let namespace: String = r.get("namespace"); - let local_name: String = r.get("local_name"); - let value: Option = r.get("value"); - (QualifiedName::new(namespace, local_name), value) - }) - .collect()) + Ok(rows.into_iter().map(row_to_prop).collect()) + } + + /// Batched variant of [`get_all`] for every file in a PROPFIND page: + /// ONE `file_id = ANY($1)` round-trip instead of N sequential queries. + /// Files with no dead properties are simply absent from the map. + pub async fn get_all_for_files( + &self, + file_ids: &[Uuid], + ) -> Result)>>, DomainError> { + self.get_all_batched("file_id", file_ids).await + } + + /// Batched variant of [`get_all`] for every subfolder in a PROPFIND page. + pub async fn get_all_for_folders( + &self, + folder_ids: &[Uuid], + ) -> Result)>>, DomainError> { + self.get_all_batched("folder_id", folder_ids).await + } + + async fn get_all_batched( + &self, + column: &str, + ids: &[Uuid], + ) -> Result)>>, DomainError> { + if ids.is_empty() { + return Ok(HashMap::new()); + } + let rows = sqlx::query(&format!( + "SELECT {column} AS resource_id, namespace, local_name, value + FROM storage.webdav_dead_properties + WHERE {column} = ANY($1)", + )) + .bind(ids) + .fetch_all(&*self.pool) + .await + .map_err(|e| { + DomainError::internal_error("DeadPropertyStore", format!("get_all_batched: {e}")) + })?; + + let mut map: HashMap)>> = HashMap::new(); + for row in rows { + let resource_id: Uuid = row.get("resource_id"); + map.entry(resource_id).or_default().push(row_to_prop(row)); + } + Ok(map) } /// Return a specific dead property, or `None` if not stored. @@ -181,16 +224,14 @@ impl DeadPropertyStore { r: ResourceRef, name: &QualifiedName, ) -> Result>, DomainError> { - let (folder_id, file_id) = split_ref(r); - let row = sqlx::query( + let (column, id) = split_ref(r); + let row = sqlx::query(&format!( "SELECT value FROM storage.webdav_dead_properties - WHERE folder_id IS NOT DISTINCT FROM $1 - AND file_id IS NOT DISTINCT FROM $2 - AND namespace = $3 - AND local_name = $4", - ) - .bind(folder_id) - .bind(file_id) + WHERE {column} = $1 + AND namespace = $2 + AND local_name = $3", + )) + .bind(id) .bind(&name.namespace) .bind(&name.name) .fetch_optional(&*self.pool) @@ -201,16 +242,23 @@ impl DeadPropertyStore { } } -/// Splits a `ResourceRef` into `(folder_id, file_id)` Option pairs for -/// binding into SQL. The unused slot is `None` so `IS NOT DISTINCT FROM` -/// matches the NULL stored in the unused column. -fn split_ref(r: ResourceRef) -> (Option, Option) { +/// Maps a `ResourceRef` onto the column that stores it plus the id to bind. +/// The column name is one of two compile-time literals — never user input — +/// so interpolating it into the SQL text is safe. +fn split_ref(r: ResourceRef) -> (&'static str, Uuid) { match r { - ResourceRef::Folder(id) => (Some(id), None), - ResourceRef::File(id) => (None, Some(id)), + ResourceRef::Folder(id) => ("folder_id", id), + ResourceRef::File(id) => ("file_id", id), } } +fn row_to_prop(r: sqlx::postgres::PgRow) -> (QualifiedName, Option) { + let namespace: String = r.get("namespace"); + let local_name: String = r.get("local_name"); + let value: Option = r.get("value"); + (QualifiedName::new(namespace, local_name), value) +} + pub fn create_dead_property_store(pool: Arc) -> Arc { Arc::new(DeadPropertyStore::new(pool)) } diff --git a/src/infrastructure/services/zip_service.rs b/src/infrastructure/services/zip_service.rs index a05886c4..b84087d5 100644 --- a/src/infrastructure/services/zip_service.rs +++ b/src/infrastructure/services/zip_service.rs @@ -52,7 +52,13 @@ enum ZipPlanEntry { /// Directory entry (Stored, zero-length body). Dir(String), /// File entry: ZIP-relative path + file id to stream from the blob store. - File { zip_path: String, file_id: String }, + /// `compression` is picked from the file's MIME type at plan time — + /// `Stored` for already-compressed media (JPEG/MP4/…), `Deflate` otherwise. + File { + zip_path: String, + file_id: String, + compression: Compression, + }, } /// Message protocol from the prefetch task to the ZIP writer. For each @@ -74,8 +80,11 @@ const PREFETCH_BUFFER_CHUNKS: usize = 64; /// /// Uses `async_zip` for fully-async archive creation. Every write (headers, /// compressed chunk data, central directory) goes through -/// `tokio::io::BufWriter` → `tokio::fs::File`, so **no Tokio worker is ever -/// blocked** by disk I/O or compression. +/// `tokio::io::BufWriter` → `tokio::fs::File`, so no Tokio worker is ever +/// blocked by disk I/O. Deflate itself DOES run inline on the writing task +/// (async_zip compresses inside `poll_write`), which is why entries whose +/// MIME says the content is already compressed are `Stored` instead — that +/// turns the archive hot path from ~1 CPU core per download into CRC + memcpy. /// /// Archive creation is a 2-stage pipeline: a prefetch task reads file /// content from the blob store ahead of the writer, so the next file's @@ -183,6 +192,9 @@ impl ZipService { plan.push(ZipPlanEntry::File { zip_path: format!("{}{}", zip_dir, file.name), file_id: file.id.to_string(), + compression: crate::common::mime_detect::zip_entry_compression( + &file.mime_type, + ), }); } } @@ -228,8 +240,12 @@ impl ZipService { } } } - ZipPlanEntry::File { zip_path, .. } => { - Self::write_prefetched_file(&mut zip, zip_path, &mut rx).await?; + ZipPlanEntry::File { + zip_path, + compression, + .. + } => { + Self::write_prefetched_file(&mut zip, zip_path, *compression, &mut rx).await?; } } } @@ -282,17 +298,19 @@ impl ZipService { } } - /// Writer stage: drains one file's prefetched chunks into a Deflate - /// ZIP entry. Peak memory stays bounded by the channel, independent - /// of individual file sizes. + /// Writer stage: drains one file's prefetched chunks into a ZIP entry + /// (`Stored` for already-compressed media, `Deflate` otherwise — see + /// `entry_compression`). Peak memory stays bounded by the channel, + /// independent of individual file sizes. async fn write_prefetched_file( zip: &mut AsyncZipWriter, zip_path: &str, + compression: Compression, rx: &mut tokio::sync::mpsc::Receiver, ) -> Result<()> { info!("Adding file to ZIP: {}", zip_path); - let entry = ZipEntryBuilder::new(zip_path.to_string().into(), Compression::Deflate); + let entry = ZipEntryBuilder::new(zip_path.to_string().into(), compression); let mut entry_writer = zip .write_entry_stream(entry) .await diff --git a/src/interfaces/api/handlers/webdav_handler.rs b/src/interfaces/api/handlers/webdav_handler.rs index 6268ddd0..67c2af38 100644 --- a/src/interfaces/api/handlers/webdav_handler.rs +++ b/src/interfaces/api/handlers/webdav_handler.rs @@ -38,6 +38,7 @@ use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; use crate::interfaces::range_requests::{not_modified_response, range_response}; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode}; +use std::collections::HashMap; use std::sync::Arc; /// Characters that MUST NOT be percent-encoded inside a URI path segment. @@ -557,7 +558,13 @@ async fn handle_propfind( created_by: None, updated_by: None, }; - let quota = state.resolve_webdav_quota(user.id, Uuid::nil()).await; + // Skip the 2-query quota resolution when the request's prop list + // never mentions quota (benches/QUOTA-PATH.md). + let quota = if propfind_request.wants_quota() { + state.resolve_webdav_quota(user.id, Uuid::nil()).await + } else { + None + }; return build_streaming_propfind_response( root_folder, None, // folder_id = None → root children (drive-root folders) @@ -597,7 +604,11 @@ async fn handle_propfind( ) .await?; let folder_id = folder.id.clone(); - let quota = state.resolve_webdav_quota(user.id, drive_id).await; + let quota = if propfind_request.wants_quota() { + state.resolve_webdav_quota(user.id, drive_id).await + } else { + None + }; return build_streaming_propfind_response( folder, Some(folder_id), @@ -663,7 +674,11 @@ async fn handle_propfind( ) .await?; let folder_id = folder.id.clone(); - let quota = state.resolve_webdav_quota(user.id, drive_id).await; + let quota = if propfind_request.wants_quota() { + state.resolve_webdav_quota(user.id, drive_id).await + } else { + None + }; return build_streaming_propfind_response( folder, Some(folder_id), @@ -788,19 +803,18 @@ async fn build_streaming_propfind_response( break; } - // Materialise dead-props for the whole page before - // we start writing — keeps the borrow checker happy - // (the writer borrows the FolderDto and the dead-props - // vec for the duration of write_folder_entry_*). - let mut subfolder_deads = Vec::with_capacity(result.items.len()); - for subfolder in &result.items { - subfolder_deads.push(folder_dead_props(&dead_props_store, subfolder).await); - } + // ONE batched dead-props query per page instead of a + // sequential per-child round-trip — the N+1 shape cost + // 1-4.5 s of pure DB chatter on a 2000-child folder + // (measured in benches/DEAD-PROPS.md). + let subfolder_deads = + folders_dead_props_map(&dead_props_store, &result.items).await; let mut chunk = Vec::with_capacity(result.items.len() * 800); { let mut w = Writer::new(&mut chunk); - for (subfolder, child_dead) in result.items.iter().zip(subfolder_deads.iter()) { + for subfolder in result.items.iter() { + let child_dead = dead_props_for(&subfolder.id, &subfolder_deads); let href = format!("{}{}/", base_href, encode_path_segment(&subfolder.name)); WebDavAdapter::write_folder_entry_with_dead_props(&mut w, subfolder, &propfind_request, &href, child_dead, quota) .map_err(|e| std::io::Error::other(e.to_string()))?; @@ -815,11 +829,17 @@ async fn build_streaming_propfind_response( page += 1; } - // Stream files in pages (user-scoped) - let mut offset: i64 = 0; + // Stream files in pages (user-scoped, keyset cursor — O(page) + // per page instead of the quadratic LIMIT/OFFSET walk). + let mut after_name: Option = None; loop { let batch: Vec = file_retrieval_service - .list_files_batch_with_perms(fid_ref, user_id, offset, PROPFIND_BATCH_SIZE) + .list_files_batch_with_perms( + fid_ref, + user_id, + after_name.as_deref(), + PROPFIND_BATCH_SIZE, + ) .await .map_err(|e| std::io::Error::other(e.to_string()))?; @@ -828,15 +848,14 @@ async fn build_streaming_propfind_response( } let batch_len = batch.len(); - let mut file_deads = Vec::with_capacity(batch_len); - for file in &batch { - file_deads.push(streamed_file_dead_props(&dead_props_store, file).await); - } + // Batched: one = ANY($1) query per 500-file page. + let file_deads = files_dead_props_map(&dead_props_store, &batch).await; let mut chunk = Vec::with_capacity(batch_len * 800); { let mut w = Writer::new(&mut chunk); - for (file, child_dead) in batch.iter().zip(file_deads.iter()) { + for file in batch.iter() { + let child_dead = dead_props_for(&file.id, &file_deads); let href = format!("{}{}", base_href, encode_path_segment(&file.name)); WebDavAdapter::write_file_entry_with_dead_props(&mut w, file, &propfind_request, &href, child_dead) .map_err(|e| std::io::Error::other(e.to_string()))?; @@ -847,7 +866,7 @@ async fn build_streaming_propfind_response( if (batch_len as i64) < PROPFIND_BATCH_SIZE { break; } - offset += batch_len as i64; + after_name = batch.last().map(|f| f.name.clone()); } } @@ -1381,20 +1400,45 @@ pub(crate) async fn folder_dead_props( .unwrap_or_default() } -/// File-leaf variant for the streaming walker (takes a `&DeadPropertyStore` -/// rather than the full `&Arc` so it can be called from inside -/// the async-stream future without cloning state). -pub(crate) async fn streamed_file_dead_props( +/// Batched dead-props fetch for a whole PROPFIND page of files: ONE +/// `file_id = ANY($1)` round-trip instead of one query per child (the old +/// per-child `streamed_file_dead_props` loop cost seconds on large folders — +/// benches/DEAD-PROPS.md). Same leniency as the single-resource helpers: +/// any failure → empty map, so the PROPFIND still emits live properties. +pub(crate) async fn files_dead_props_map( store: &DeadPropertyStore, - file: &FileDto, -) -> Vec<(QualifiedName, Option)> { - let Ok(file_id) = Uuid::parse_str(&file.id) else { - return Vec::new(); - }; - store - .get_all(ResourceRef::File(file_id)) - .await - .unwrap_or_default() + files: &[FileDto], +) -> HashMap)>> { + let ids: Vec = files + .iter() + .filter_map(|f| Uuid::parse_str(&f.id).ok()) + .collect(); + store.get_all_for_files(&ids).await.unwrap_or_default() +} + +/// Folder-page variant of [`files_dead_props_map`]. +pub(crate) async fn folders_dead_props_map( + store: &DeadPropertyStore, + folders: &[FolderDto], +) -> HashMap)>> { + let ids: Vec = folders + .iter() + .filter_map(|f| Uuid::parse_str(&f.id).ok()) + .collect(); + store.get_all_for_folders(&ids).await.unwrap_or_default() +} + +/// Looks up one resource's dead props in a batched map (resources with no +/// dead properties are absent from the map → empty slice). +pub(crate) fn dead_props_for<'a>( + id: &str, + map: &'a HashMap)>>, +) -> &'a [(QualifiedName, Option)] { + Uuid::parse_str(id) + .ok() + .and_then(|u| map.get(&u)) + .map(|v| v.as_slice()) + .unwrap_or(&[]) } /// A single condition inside a `List` of the WebDAV `If:` header diff --git a/src/interfaces/nextcloud/basic_auth_middleware.rs b/src/interfaces/nextcloud/basic_auth_middleware.rs index c9621c0e..c781cf74 100644 --- a/src/interfaces/nextcloud/basic_auth_middleware.rs +++ b/src/interfaces/nextcloud/basic_auth_middleware.rs @@ -5,11 +5,36 @@ use axum::{ response::{IntoResponse, Response}, }; use base64::Engine; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; +use std::time::Duration; +use crate::application::dtos::folder_dto::FolderDto; use crate::common::di::AppState; use crate::interfaces::middleware::auth::CurrentUser; +/// Markerless-chroot cache: default-drive root folder id → `FolderDto`. +/// +/// This middleware wraps EVERY protected NextCloud route (DAV files, +/// per-chunk uploads, trashbin, previews, avatars, OCS polls). With the +/// app-password verification already cached, the chroot resolution was the +/// last per-request DB work: `find_default_for_user` (now cached in +/// `DrivePgRepository`) plus this folder-by-PK fetch. A desktop sync run +/// issues hundreds of these per minute for a value that changes only on a +/// root-folder rename — the 30 s TTL bounds that staleness (mirrors +/// `drive_role_cache` / the default-drive cache; measured in +/// `benches/CHROOT-CACHE.md`). +/// +/// Only the MARKERLESS branch is cached: it targets the caller's own +/// default drive root, so no per-request authorization decision is being +/// skipped. The drive-marker branch keeps its `get_folder_with_perms` +/// check on every request. +static NC_CHROOT_CACHE: LazyLock> = LazyLock::new(|| { + moka::sync::Cache::builder() + .max_capacity(100_000) + .time_to_live(Duration::from_secs(30)) + .build() +}); + #[derive(Debug, thiserror::Error)] pub enum NextcloudAuthError { #[error("Unauthorized")] @@ -191,12 +216,24 @@ pub async fn basic_auth_middleware( .find_default_for_user(current_user.id) .await { - Ok(drive_with_name) => state - .applications - .folder_service - .get_folder(&drive_with_name.drive.root_folder_id.to_string()) - .await - .ok(), + Ok(drive_with_name) => { + let root_id = drive_with_name.drive.root_folder_id; + match NC_CHROOT_CACHE.get(&root_id) { + Some(cached) => Some(cached), + None => { + let fetched = state + .applications + .folder_service + .get_folder(&root_id.to_string()) + .await + .ok(); + if let Some(f) = &fetched { + NC_CHROOT_CACHE.insert(root_id, f.clone()); + } + fetched + } + } + } Err(_) => None, } } diff --git a/src/interfaces/nextcloud/report_handler.rs b/src/interfaces/nextcloud/report_handler.rs index 609180ce..68ff9f14 100644 --- a/src/interfaces/nextcloud/report_handler.rs +++ b/src/interfaces/nextcloud/report_handler.rs @@ -21,7 +21,9 @@ use crate::application::ports::folder_ports::FolderUseCase; use crate::application::ports::inbound::SearchUseCase; use crate::common::di::AppState; use crate::domain::entities::file::File; -use crate::interfaces::api::handlers::webdav_handler::{file_dead_props, folder_dead_props}; +use crate::interfaces::api::handlers::webdav_handler::{ + dead_props_for, files_dead_props_map, folders_dead_props_map, +}; use crate::interfaces::errors::AppError; use crate::interfaces::nextcloud::webdav_handler::{ batch_resolve_ids, format_oc_id, nc_href, write_file_response, write_folder_response, @@ -160,6 +162,11 @@ async fn handle_filter_files( write_multistatus_start(&mut xml)?; + // Batched dead-props: one = ANY($1) query per type, not one per + // result (benches/DEAD-PROPS.md). + let file_deads = files_dead_props_map(&state.webdav_dead_props, &files).await; + let folder_deads = folders_dead_props_map(&state.webdav_dead_props, &folders).await; + // Keep main's batched-resolution structure (one batch query // per type, not 2N round-trips). Hrefs use `url_user` so the // multi-drive `~{drive}` form is echoed back to the client; @@ -179,7 +186,7 @@ async fn handle_filter_files( let href = nc_href(url_user, subpath); let fid = file_id_map.get(&file.id).copied(); let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); - let dead = file_dead_props(&state, file).await; + let dead = dead_props_for(&file.id, &file_deads); write_file_response( &mut xml, file, @@ -187,7 +194,7 @@ async fn handle_filter_files( (fid, oc_id.as_deref()), &user.username, &favorite_ids, - &dead, + dead, ) .map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?; } @@ -205,7 +212,7 @@ async fn handle_filter_files( let href = format!("{}/", nc_href(url_user, subpath)); let fid = folder_id_map.get(&folder.id).copied(); let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); - let dead = folder_dead_props(&state.webdav_dead_props, folder).await; + let dead = dead_props_for(&folder.id, &folder_deads); write_folder_response( &mut xml, folder, @@ -217,7 +224,7 @@ async fn handle_filter_files( // PROPFIND on a specific collection — quota isn't // meaningful here (see `AppState::resolve_webdav_quota`). None, - &dead, + dead, ) .map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?; } @@ -301,6 +308,11 @@ async fn handle_search( write_multistatus_start(&mut xml)?; + // Batched dead-props: one = ANY($1) query per type, not one per + // result (benches/DEAD-PROPS.md). + let file_deads = files_dead_props_map(&state.webdav_dead_props, &files).await; + let folder_deads = folders_dead_props_map(&state.webdav_dead_props, &folders).await; + // Files. for file in &files { let Some(subpath) = strip_home_prefix(chroot, &file.path, home_prefix) else { @@ -315,7 +327,7 @@ async fn handle_search( let href = nc_href(url_user, subpath); let fid = file_id_map.get(&file.id).copied(); let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); - let dead = file_dead_props(&state, file).await; + let dead = dead_props_for(&file.id, &file_deads); write_file_response( &mut xml, file, @@ -323,7 +335,7 @@ async fn handle_search( (fid, oc_id.as_deref()), &user.username, &favorite_ids, - &dead, + dead, ) .map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?; } @@ -342,7 +354,7 @@ async fn handle_search( let href = format!("{}/", nc_href(url_user, subpath)); let fid = folder_id_map.get(&folder.id).copied(); let oc_id = fid.map(|id| format_oc_id(id, file_id_svc)); - let dead = folder_dead_props(&state.webdav_dead_props, folder).await; + let dead = dead_props_for(&folder.id, &folder_deads); write_folder_response( &mut xml, folder, @@ -354,7 +366,7 @@ async fn handle_search( // PROPFIND on a specific collection — quota isn't // meaningful here (see `AppState::resolve_webdav_quota`). None, - &dead, + dead, ) .map_err(|e| AppError::internal_error(format!("XML write error: {}", e)))?; } diff --git a/src/interfaces/nextcloud/webdav_handler.rs b/src/interfaces/nextcloud/webdav_handler.rs index 6ecace2e..0bd771f4 100644 --- a/src/interfaces/nextcloud/webdav_handler.rs +++ b/src/interfaces/nextcloud/webdav_handler.rs @@ -30,7 +30,8 @@ use crate::domain::services::authorization::{Permission, Resource, Subject}; use crate::infrastructure::services::path_resolver_service::ResolvedResource; use crate::infrastructure::services::webdav_dead_property_store::ResourceRef; use crate::interfaces::api::handlers::webdav_handler::{ - PROPFIND_BATCH_SIZE, file_dead_props, folder_dead_props, streamed_file_dead_props, + PROPFIND_BATCH_SIZE, dead_props_for, file_dead_props, files_dead_props_map, folder_dead_props, + folders_dead_props_map, }; use crate::interfaces::errors::AppError; use crate::interfaces::range_requests::{not_modified_response, range_response}; @@ -297,9 +298,10 @@ async fn handle_propfind( .map_err(|e| AppError::bad_request(format!("Failed to read body: {}", e)))?; // Parse (and thereby validate) the PROPFIND body. The NC response - // always emits the full property set, so the parsed request is not - // consulted further — but malformed XML must still fail with 400. - let _propfind = if body_bytes.is_empty() { + // always emits the full property set; the parsed request is consulted + // only to skip the quota DB round-trips when the client's explicit + // prop list never names a quota prop. Malformed XML still fails 400. + let propfind = if body_bytes.is_empty() { PropFindRequest { prop_find_type: crate::application::adapters::webdav_adapter::PropFindType::AllProp, } @@ -341,7 +343,13 @@ async fn handle_propfind( // function's username arg. Refining the owner-id usages // back to the canonical username is deferred to the // NcSession commit. - let quota = state.resolve_webdav_quota(user.id, chroot.drive_id).await; + // Explicit prop lists that never name a quota prop skip the + // 2-query quota resolution (benches/QUOTA-PATH.md). + let quota = if propfind.wants_quota() { + state.resolve_webdav_quota(user.id, chroot.drive_id).await + } else { + None + }; Ok(build_nc_streaming_propfind( state.clone(), folder, @@ -1519,11 +1527,17 @@ fn build_nc_streaming_propfind( // ── Children (only if Depth != 0) ──────────────────────────── if depth != "0" { - // Files in pages. - let mut offset: i64 = 0; + // Files in pages (keyset cursor — O(page) per page instead of + // the quadratic LIMIT/OFFSET walk). + let mut after_name: Option = None; loop { let batch = file_service - .list_files_batch_with_perms(Some(&folder.id), user_id, offset, PROPFIND_BATCH_SIZE) + .list_files_batch_with_perms( + Some(&folder.id), + user_id, + after_name.as_deref(), + PROPFIND_BATCH_SIZE, + ) .await .map_err(|e| std::io::Error::other(e.to_string()))?; if batch.is_empty() { @@ -1541,15 +1555,15 @@ fn build_nc_streaming_propfind( }; let file_uuids: Vec = batch.iter().map(|f| f.id.clone()).collect(); let (file_id_map, _) = batch_resolve_ids(file_id_svc, &file_uuids, &[]).await; - let mut file_deads = Vec::with_capacity(batch_len); - for file in &batch { - file_deads.push(streamed_file_dead_props(&state.webdav_dead_props, file).await); - } + // One batched dead-props query per page, not one per child + // (benches/DEAD-PROPS.md). + let file_deads = files_dead_props_map(&state.webdav_dead_props, &batch).await; let mut chunk = Vec::with_capacity(batch_len * 1024); { let mut xml = Writer::new(&mut chunk); - for (file, dead) in batch.iter().zip(file_deads.iter()) { + for file in batch.iter() { + let dead = dead_props_for(&file.id, &file_deads); let child_sub = if subpath.is_empty() { file.name.clone() } else { @@ -1567,7 +1581,7 @@ fn build_nc_streaming_propfind( if (batch_len as i64) < PROPFIND_BATCH_SIZE { break; } - offset += batch_len as i64; + after_name = batch.last().map(|f| f.name.clone()); } // Subfolders in pages — also collections, same trailing-slash rule. @@ -1594,15 +1608,15 @@ fn build_nc_streaming_propfind( }; let folder_uuids: Vec = result.items.iter().map(|sf| sf.id.clone()).collect(); let (_, sub_id_map) = batch_resolve_ids(file_id_svc, &[], &folder_uuids).await; - let mut sub_deads = Vec::with_capacity(result.items.len()); - for sf in &result.items { - sub_deads.push(folder_dead_props(&state.webdav_dead_props, sf).await); - } + // Batched — see benches/DEAD-PROPS.md. + let sub_deads = + folders_dead_props_map(&state.webdav_dead_props, &result.items).await; let mut chunk = Vec::with_capacity(result.items.len() * 1024); { let mut xml = Writer::new(&mut chunk); - for (sf, dead) in result.items.iter().zip(sub_deads.iter()) { + for sf in result.items.iter() { + let dead = dead_props_for(&sf.id, &sub_deads); let child_sub = if subpath.is_empty() { sf.name.clone() } else { diff --git a/src/interfaces/web/mod.rs b/src/interfaces/web/mod.rs index c0ca5eae..2239a1a4 100644 --- a/src/interfaces/web/mod.rs +++ b/src/interfaces/web/mod.rs @@ -46,10 +46,23 @@ pub fn create_web_routes() -> Router> { let static_path = resolve_static_path(&config); // SPA fallback: serve the file if it exists, else the app shell. - let spa = ServeDir::new(&static_path).fallback(ServeFile::new(static_path.join("index.html"))); + // + // `precompressed_*`: if the frontend build emitted a sibling `.br`/`.gz` + // (frontend/scripts/precompress.mjs runs at build time), serve those + // bytes directly with the right Content-Encoding instead of re-running + // Brotli over the same immutable bundle on EVERY request — the + // `CompressionLayer` below then skips the already-encoded response and + // remains only the fallback for assets without a precompressed sibling + // (benches/STATIC-PRECOMPRESSED.md). + let spa = ServeDir::new(&static_path) + .precompressed_br() + .precompressed_gzip() + .fallback(ServeFile::new(static_path.join("index.html"))); // Hashed, immutable assets (SvelteKit emits these under /_app/immutable). - let app_immutable = ServeDir::new(static_path.join("_app").join("immutable")); + let app_immutable = ServeDir::new(static_path.join("_app").join("immutable")) + .precompressed_br() + .precompressed_gzip(); Router::new() .nest_service( @@ -60,7 +73,17 @@ pub fn create_web_routes() -> Router> { )), ) .fallback_service(spa) - .layer(CompressionLayer::new().br(true).gzip(true)) + // Fallback compression for assets without a precompressed sibling. + // Quality 4, NOT the default: the default maps to Brotli q11 — + // ~1.3 s of CPU per 700 KiB bundle per request (measured in + // benches/STATIC-PRECOMPRESSED.md; the .br siblings above carry the + // real q11 bytes, paid once at build time). + .layer( + CompressionLayer::new() + .quality(tower_http::CompressionLevel::Precise(4)) + .br(true) + .gzip(true), + ) // `if_not_present` so the immutable assets above keep their long cache; // the shell itself must always revalidate so a deploy can't pin a stale // app in browsers. diff --git a/src/main.rs b/src/main.rs index 6c2dc005..d2d59541 100644 --- a/src/main.rs +++ b/src/main.rs @@ -880,7 +880,18 @@ async fn run() -> Result<(), Box> { // ── file-body downloads carry Content-Disposition (see above) ── .and(NotForDownloads); - app = app.layer(CompressionLayer::new().compress_when(predicate)); + // Explicit quality: the layer's default maps to Brotli QUALITY 11 + // (async-compression Level::Default → BrotliEncoderParams::default(), + // brotli-8.0.2 encode.rs:323) — a deploy-grade setting that cost + // ~90 ms of CPU per 64 KiB JSON response. Level 4 emits ~15 % more + // bytes at ~1 % of the CPU (0.9 ms) — measured in + // benches/STATIC-PRECOMPRESSED.md. Applies to gzip too (level 4, + // the classic dynamic-content setting). + app = app.layer( + CompressionLayer::new() + .quality(tower_http::CompressionLevel::Precise(4)) + .compress_when(predicate), + ); } // ── Security headers ───────────────────────────────────────────────── From 82ee7da0d2860fe16cd78cbf1f332bace7a5cdca Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 16:50:07 +0000 Subject: [PATCH 2/2] perf: serve ranges from RAM cache, stream ZIPs, overlap ingest settle, O(1) chunk gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 of benchmark-gated optimizations (benches/ROUND2.md; every change gated by a before/after in examples/bench_round2.rs — an AFTER that did not beat its BEFORE was to be rolled back; none needed it): - Range requests (REST/DAV/shares) answered from the moka content cache for sub-10MB files: PG resolve + open/seek/read -> Bytes::slice. 256KiB seeks: 1,730/s -> 3.7M/s (p50 552us -> 0.15us). - Streaming folder/share ZIPs via tokio duplex: TTFB no longer scales with archive size (326ms -> 0.4ms on 192MiB corpus; total also faster). Content-Length dropped (size unknown up front). - NC chunked-upload per-PUT gate: O(k) directory scan+stat -> in-RAM per-session counter (lazy rebuild on cold start). 1,000-chunk upload gate cost: 33.1s -> 0.09s cumulative. - Delta download + commit-verify now use the CDC path's buffered(read_prefetch) read-ahead: 64-chunk drain at 5ms open latency 440ms -> 51ms; order preserved. - CDC ingest settles batches on a spawned task (depth-1 pipeline) so the source stream keeps flowing during PG pin + backend writes; rollback ledger shared + lock-serialized so compensation stays exact on cancellation. 512MiB paced ingest: 60-69 -> 74-75 MB/s. OXICLOUD_INGEST_OVERLAP=0 restores inline settling (ops/bench hatch). - Frontend: instant-upload BLAKE3 hashing moved off the main thread to a bounded Web Worker pool (File handles by reference); vitest gate asserts the pool beats sequential (first gate draft posting buffers was 2.6x slower and was rewritten — copies dominated). Validation: cargo fmt + clippy -D warnings clean; 514 unit + 544 integration tests green; 270 frontend tests green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w --- Cargo.toml | 8 + benches/ROUND2.md | 107 +++++ examples/bench_round2.rs | 400 ++++++++++++++++++ .../api/endpoints/deltaUpload.hash.test.ts | 87 ++++ frontend/src/lib/api/endpoints/deltaUpload.ts | 61 ++- frontend/static/workers/hashWorker.js | 40 ++ src/application/ports/file_ports.rs | 11 + .../services/delta_upload_service.rs | 6 + .../services/file_retrieval_service.rs | 67 ++- src/infrastructure/services/dedup_service.rs | 195 +++++++-- .../nextcloud_chunked_upload_service.rs | 74 +++- src/infrastructure/services/zip_service.rs | 169 +++++--- .../api/handlers/delta_upload_handler.rs | 41 +- src/interfaces/api/handlers/file_handler.rs | 17 +- src/interfaces/api/handlers/folder_handler.rs | 50 +-- src/interfaces/api/handlers/share_handler.rs | 42 +- src/interfaces/nextcloud/uploads_handler.rs | 25 +- src/interfaces/range_requests.rs | 40 +- 18 files changed, 1245 insertions(+), 195 deletions(-) create mode 100644 benches/ROUND2.md create mode 100644 examples/bench_round2.rs create mode 100644 frontend/src/lib/api/endpoints/deltaUpload.hash.test.ts create mode 100644 frontend/static/workers/hashWorker.js diff --git a/Cargo.toml b/Cargo.toml index d500f80c..ec5c9b5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -220,6 +220,14 @@ name = "bench_static_precompress" path = "examples/bench_static_precompress.rs" required-features = ["bench"] +# Round-2 battery: range-from-cache, NC chunk gate, delta prefetch, ingest +# overlap (real store_from_stream; run with OXICLOUD_INGEST_OVERLAP=0/1), +# ZIP streaming TTFB. Sections 1 and 4 need Postgres. +[[example]] +name = "bench_round2" +path = "examples/bench_round2.rs" +required-features = ["bench"] + # Video thumbnail benchmark — Option B (server-side ffmpeg frame → WebP). Needs # `ffmpeg` on PATH (libx264/libx265/libvpx-vp9 to synthesize the test corpus). [[example]] diff --git a/benches/ROUND2.md b/benches/ROUND2.md new file mode 100644 index 00000000..12b4a70c --- /dev/null +++ b/benches/ROUND2.md @@ -0,0 +1,107 @@ +# Round 2 — read path, upload path, archives (before/after gates) + +Five backend changes + one frontend change, each gated by a before/after +benchmark (`examples/bench_round2.rs`; frontend gate in +`frontend/src/lib/api/endpoints/deltaUpload.hash.test.ts`). Rule of the +round: an AFTER that doesn't beat its BEFORE gets rolled back — none did. + +Reproduce: + +```bash +BENCH_SECTIONS=1,2,3,5 cargo run --release --features bench --example bench_round2 +OXICLOUD_INGEST_OVERLAP=0 BENCH_SECTIONS=4 cargo run --release --features bench --example bench_round2 +OXICLOUD_INGEST_OVERLAP=1 BENCH_SECTIONS=4 cargo run --release --features bench --example bench_round2 +cd frontend && npx vitest run src/lib/api/endpoints/deltaUpload.hash.test.ts +``` + +## [1] Range requests served from the content cache — 2,156× + +Media players and PDF viewers fetch files *exclusively* via Range requests +(a `bytes=0-` probe, then seeks). All three range paths (REST, DAV helper, +public shares) went straight to `get_file_range_stream`: a PG blob-hash +resolve + chunk open/seek/read per seek — even when the whole sub-10 MB blob +sat in the moka content cache as contiguous `Bytes`. +`FileRetrievalService::get_file_range_preloaded` now answers from the cache +(`Bytes::slice` = refcount bump; a miss populates it via the same +single-flight loader Tier 1 uses, so one probe warms every later seek). + +| per 256 KiB seek (6 MiB file) | seeks/s | p50 µs | p99 µs | +|-------------------------------|--------:|-------:|-------:| +| BEFORE — PG + open/seek/read | 1,730 | 552.5 | 818.8 | +| AFTER — cache hit + slice | 3,730,560 | 0.15 | 2.85 | + +## [2] NC chunked-upload gate: O(N²) directory scan → O(1) counter — 357× + +`handle_put_chunk` recomputed "session bytes so far" on EVERY chunk PUT by +listing the session directory and stat-ing every existing chunk — chunk k +scans k files; a 1,000-chunk (10 GB) upload does ~500k stats. +`NextcloudChunkedUploadService` now keeps an in-RAM per-session counter +(seeded on MKCOL, bumped per accepted chunk, dropped on cleanup/overwrite, +lazily rebuilt from the listing on cold start — crash semantics unchanged). + +Cumulative gate cost across a 1,000-chunk upload: **33,063 ms → 93 ms**. + +## [3] Delta download / commit-verify read-ahead — 8.7× (latency-bound) + +`delta_download_chunks` and `hash_chunk_sequence` drained chunks strictly +sequentially — every chunk-open's round-trip paid serially — while the main +CDC download path already overlaps opens with `buffered(read_prefetch)`. +Both now use the same combinator (order preserved — `buffered` yields in +input order). + +64-chunk drain with 5 ms per-open latency (object-store model): +**440 ms → 51 ms**. On local disk the same combinator measured +7–12 % +(benches/BLOB-PREFETCH.md). + +## [4] CDC ingest: settle overlapped with reading — +7–25 % + +`ingest_chunks_from_stream` awaited each batch settle (PG pin round-trip + +up to 8 MiB of backend writes) INLINE — the HTTP source was not polled at +all during the settle, so read and settle phases alternated instead of +overlapping. The settle now runs on a spawned task (depth-1 pipeline) that +records into the guard's shared, lock-serialized state — rollback stays +exact even if the request future is dropped mid-settle. +`OXICLOUD_INGEST_OVERLAP=0` restores the inline behaviour (the bench's +BEFORE side, and an ops escape hatch). + +512 MiB unique-content ingest, source paced at 300 MB/s, two reps: +**60 / 69 MB/s (inline) → 75 / 74 MB/s (overlapped)**. + +## [5] Streaming ZIP: constant time-to-first-byte — 779× on this corpus + +`create_folder_zip` built the ENTIRE archive into a temp file before the +handler sent byte one — TTFB grew with folder size (a multi-GB folder = +minutes of "waiting for server"). `create_folder_zip_stream` plans inline +(planning errors still surface as proper HTTP errors), then writes the +archive on a spawned task through `tokio::io::duplex`, streaming bytes as +they are produced. Folder downloads and public-share ZIPs both use it; a +mid-archive blob error truncates the stream (no central directory → clients +detect corruption) — the standard streamed-ZIP tradeoff. Content-Length is +no longer sent (size unknown up front). + +48 × 4 MiB media corpus: TTFB **326.1 ms → 0.4 ms**; total wall also +improved (484 ms → 55 ms — no disk round-trip through the temp file). +TTFB in BEFORE scales linearly with archive size; AFTER is constant. + +## [6] Frontend: instant-upload hashing on a worker pool + +`resolveOwnedHashes` hashed every small file of a drop sequentially on the +MAIN THREAD (synchronous WASM BLAKE3 per file) before any upload lane +started — seconds of UI jank on large drops. Hashing now fans out over a +bounded pool of dedicated Web Workers (`static/workers/hashWorker.js`, +`File` handles passed by reference, reads happen inside the worker), with +the old inline loop kept as fallback where `Worker` is unavailable. + +Architecture gate (node worker_threads, read+hash 24 × 4 MiB, file +references — faithful to the browser shape): 3-lane pool beats the +sequential loop; asserted by `deltaUpload.hash.test.ts` so a regression +fails CI. First model of this gate (posting BUFFERS instead of file +references) was 2.6× SLOWER — structured-clone copies dominated — and was +rewritten; kept here as a reminder that the gate must model the real +data-flow. + +## Skipped this round + +- **Swimlane (group-by) view virtualization** — needs interactive browser + measurement (frame times while scrolling) that this environment can't + produce; deferred rather than shipped unverified. diff --git a/examples/bench_round2.rs b/examples/bench_round2.rs new file mode 100644 index 00000000..2e79bbe2 --- /dev/null +++ b/examples/bench_round2.rs @@ -0,0 +1,400 @@ +//! Round-2 benchmark battery — five before/after gates in one binary. +//! +//! Each section isolates exactly what its change touches; a section whose +//! AFTER does not beat its BEFORE is grounds for rolling that change back. +//! +//! [1] range-cache — per-seek: PG resolve + open/seek/read vs moka hit + Bytes::slice +//! [2] nc-chunk-gate — per-PUT session-bytes gate: dir scan+stat vs counter +//! [3] delta-prefetch — 64-chunk drain: sequential opens vs buffered(8) (5 ms open latency) +//! [4] ingest-overlap — real store_from_stream, paced source: OXICLOUD_INGEST_OVERLAP=0 vs 1 +//! [5] zip-stream — time-to-first-byte: temp-file build vs duplex streaming +//! +//! Run (needs Postgres for [1] and [4]; reads DATABASE_URL from .env): +//! cargo run --release --features bench --example bench_round2 +//! Select sections: BENCH_SECTIONS="1,2,3,4,5" + +use std::env; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use bytes::Bytes; +use futures::{StreamExt, TryStreamExt, stream}; +use sqlx::postgres::PgPoolOptions; +use uuid::Uuid; + +fn env_or(key: &str, default: T) -> T { + env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +fn median(mut xs: Vec) -> f64 { + xs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + xs[xs.len() / 2] +} + +fn pct(sorted: &[f64], p: f64) -> f64 { + if sorted.is_empty() { + return 0.0; + } + sorted[((sorted.len() as f64 * p) as usize).min(sorted.len() - 1)] +} + +fn fill_random(buf: &mut [u8], seed: &mut u64) { + for chunk in buf.chunks_mut(8) { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + let b = seed.wrapping_mul(0x2545F4914F6CDD1D).to_le_bytes(); + let n = chunk.len(); + chunk.copy_from_slice(&b[..n]); + } +} + +// ── [1] range-cache ───────────────────────────────────────────────────────── +async fn section_range_cache(url: &str) { + println!("\n== [1] range-cache: per-seek cost, 256 KiB ranges over a 6 MiB media file =="); + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(url) + .await + .expect("pg"); + + // Seed: drive→folder→file row (the BEFORE path resolves blob_hash by id) + // plus the blob bytes on disk for the open/seek/read. + let mut tx = pool.begin().await.expect("tx"); + let drive_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.drives (kind, quota_bytes) VALUES ('shared', NULL) RETURNING id", + ) + .fetch_one(&mut *tx) + .await + .unwrap(); + let folder_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.folders (name, path, lpath, drive_id) + VALUES ('bench_range', '/bench_range', 'bench_range', $1) RETURNING id", + ) + .bind(drive_id) + .fetch_one(&mut *tx) + .await + .unwrap(); + sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2") + .bind(folder_id) + .bind(drive_id) + .execute(&mut *tx) + .await + .unwrap(); + tx.commit().await.unwrap(); + + let blob_hash = "benchrange000000000000000000000000000000000000000000000000000000"; + let file_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.files (name, folder_id, blob_hash, size, mime_type, drive_id) + VALUES ('video.mp4', $1, $2, 6291456, 'video/mp4', $3) RETURNING id", + ) + .bind(folder_id) + .bind(blob_hash) + .bind(drive_id) + .fetch_one(&pool) + .await + .unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let mut data = vec![0u8; 6 * 1024 * 1024]; + let mut seed = 7u64; + fill_random(&mut data, &mut seed); + let blob_path = dir.path().join("blob"); + std::fs::write(&blob_path, &data).unwrap(); + + // AFTER: warm content cache keyed by hash. + let cache: moka::sync::Cache = moka::sync::Cache::new(1000); + cache.insert(blob_hash.to_string(), Bytes::from(data.clone())); + + let secs = 3u64; + let range_len = 256 * 1024usize; + for mode in ["BEFORE", "AFTER"] { + let deadline = Instant::now() + Duration::from_secs(secs); + let mut lats = Vec::new(); + let mut off = 0usize; + while Instant::now() < deadline { + let t = Instant::now(); + if mode == "BEFORE" { + // 1. resolve blob hash by file id (the real query shape) + let _h: String = + sqlx::query_scalar("SELECT blob_hash FROM storage.files WHERE id = $1") + .bind(file_id) + .fetch_one(&pool) + .await + .unwrap(); + // 2. open + seek + read the range (manifest lookup is already + // a moka hit post-round-1, so it's omitted on both sides) + use tokio::io::{AsyncReadExt, AsyncSeekExt}; + let mut f = tokio::fs::File::open(&blob_path).await.unwrap(); + f.seek(std::io::SeekFrom::Start(off as u64)).await.unwrap(); + let mut buf = vec![0u8; range_len]; + f.read_exact(&mut buf).await.unwrap(); + std::hint::black_box(&buf); + } else { + let bytes = cache.get(blob_hash).unwrap(); + let slice = bytes.slice(off..off + range_len); + std::hint::black_box(&slice); + } + lats.push(t.elapsed().as_secs_f64() * 1e6); + off = (off + range_len) % (data.len() - range_len); + } + lats.sort_by(|a, b| a.partial_cmp(b).unwrap()); + println!( + " {:<7} {:>9.0} seeks/s p50 {:>8.2} µs p99 {:>8.2} µs", + mode, + lats.len() as f64 / secs as f64, + pct(&lats, 0.5), + pct(&lats, 0.99), + ); + } + + let _ = sqlx::query("DELETE FROM storage.drives WHERE id = $1") + .bind(drive_id) + .execute(&pool) + .await; +} + +// ── [2] nc-chunk-gate ─────────────────────────────────────────────────────── +async fn section_nc_chunk_gate() { + println!("\n== [2] nc-chunk-gate: cumulative gate cost across a 1000-chunk upload =="); + let dir = tempfile::tempdir().unwrap(); + let session = dir.path().join("alice").join("upload-1"); + tokio::fs::create_dir_all(&session).await.unwrap(); + + let chunks: usize = env_or("BENCH_CHUNKS", 1000); + // BEFORE: every PUT lists the dir and stats every existing chunk. + let t0 = Instant::now(); + for k in 0..chunks { + // gate for chunk k: scan the k existing chunks + let mut total = 0u64; + let mut rd = tokio::fs::read_dir(&session).await.unwrap(); + while let Some(e) = rd.next_entry().await.unwrap() { + total += e.metadata().await.unwrap().len(); + } + std::hint::black_box(total); + // accept the chunk (tiny file; the write cost is identical on both + // sides so it cancels out — kept for realistic dirent counts) + tokio::fs::write(session.join(format!("{k:05}")), b"x") + .await + .unwrap(); + } + let before = t0.elapsed().as_secs_f64() * 1000.0; + + // Reset dir. + tokio::fs::remove_dir_all(&session).await.unwrap(); + tokio::fs::create_dir_all(&session).await.unwrap(); + + // AFTER: O(1) counter (moka read + insert per PUT). + let counter: moka::sync::Cache = moka::sync::Cache::new(10); + counter.insert("s".into(), 0); + let t0 = Instant::now(); + for k in 0..chunks { + let total = counter.get("s").unwrap(); + std::hint::black_box(total); + tokio::fs::write(session.join(format!("{k:05}")), b"x") + .await + .unwrap(); + counter.insert("s".into(), total + 1); + } + let after = t0.elapsed().as_secs_f64() * 1000.0; + + println!( + " BEFORE dir-scan gate: {before:>9.1} ms total AFTER counter gate: {after:>9.1} ms total ({:.1}x)", + before / after + ); + println!(" (gate work alone; chunk-write cost included identically on both sides)"); +} + +// ── [3] delta-prefetch ────────────────────────────────────────────────────── +async fn section_delta_prefetch() { + println!( + "\n== [3] delta-prefetch: 64-chunk drain, 5 ms per-open latency (object-store model) ==" + ); + let n_chunks = 64usize; + let chunk_kb = 256usize; + let mut seed = 11u64; + let mut payload = vec![0u8; chunk_kb * 1024]; + fill_random(&mut payload, &mut seed); + let payload = Bytes::from(payload); + + // One "chunk open" = latency + a 4-frame byte stream (the shape the + // handler drains). Sequential = old; buffered(8) = new combinator. + let open = |p: Bytes| async move { + tokio::time::sleep(Duration::from_millis(5)).await; + Ok::<_, std::io::Error>(stream::iter( + p.chunks(64 * 1024) + .map(|c| Ok::(Bytes::copy_from_slice(c))) + .collect::>(), + )) + }; + + for (label, prefetch) in [("BEFORE sequential", 1usize), ("AFTER buffered(8)", 8)] { + let t0 = Instant::now(); + let mut drained = 0u64; + let mut s = stream::iter(vec![payload.clone(); n_chunks]) + .map(&open) + .buffered(prefetch) + .try_flatten(); + while let Some(part) = s.next().await { + drained += part.unwrap().len() as u64; + } + let ms = t0.elapsed().as_secs_f64() * 1000.0; + println!(" {label}: {ms:>8.1} ms for {} MiB", drained / 1024 / 1024); + } + println!(" (local-disk gain for the same combinator: +7-12% — benches/BLOB-PREFETCH.md)"); +} + +// ── [4] ingest-overlap ────────────────────────────────────────────────────── +async fn section_ingest_overlap(url: &str) { + println!("\n== [4] ingest-overlap: real store_from_stream, source paced at 300 MB/s =="); + println!( + " (mode fixed per process by OXICLOUD_INGEST_OVERLAP — run twice; current = {})", + std::env::var("OXICLOUD_INGEST_OVERLAP").unwrap_or_else(|_| "1/default".into()) + ); + use oxicloud::infrastructure::services::dedup_service::DedupService; + use oxicloud::infrastructure::services::local_blob_backend::LocalBlobBackend; + + let pool = Arc::new( + PgPoolOptions::new() + .max_connections(10) + .connect(url) + .await + .expect("pg"), + ); + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(LocalBlobBackend::new(dir.path())); + use oxicloud::application::ports::blob_storage_ports::BlobStorageBackend as _; + backend.initialize().await.expect("init backend"); + let svc = DedupService::new(backend, pool.clone(), pool.clone()); + + let total_mb: usize = env_or("BENCH_INGEST_MB", 512); + let pace_mbps: f64 = env_or("BENCH_PACE_MBPS", 300.0); + let frame = 256 * 1024usize; + let mut seed = std::process::id() as u64 | 0xABCD << 32; // unique content per run — no dedup hits + let frames: Vec = (0..total_mb * 1024 * 1024 / frame) + .map(|_| { + let mut b = vec![0u8; frame]; + fill_random(&mut b, &mut seed); + Bytes::from(b) + }) + .collect(); + let frame_interval = Duration::from_secs_f64(frame as f64 / (pace_mbps * 1e6)); + + let t0 = Instant::now(); + let source = stream::iter(frames.into_iter().map(Ok::)).then( + move |f| async move { + tokio::time::sleep(frame_interval).await; + f + }, + ); + let result = svc.store_from_stream(source, None).await.expect("ingest"); + let secs = t0.elapsed().as_secs_f64(); + println!( + " ingested {} MiB in {:.2} s → {:.0} MB/s (blob {})", + total_mb, + secs, + total_mb as f64 / secs, + &result.hash()[..12], + ); + // Cleanup: release the reference so GC can reap the bench blobs. + let _ = svc.remove_reference(result.hash()).await; +} + +// ── [5] zip-stream ────────────────────────────────────────────────────────── +async fn section_zip_stream() { + println!("\n== [5] zip-stream: time-to-first-byte, 48 x 4 MiB media corpus =="); + use async_zip::base::write::ZipFileWriter; + use async_zip::{Compression, ZipEntryBuilder}; + use futures::io::AsyncWriteExt as _; + + let files: usize = env_or("BENCH_ZIP_FILES", 48); + let mb: usize = env_or("BENCH_ZIP_MB", 4); + let mut seed = 13u64; + let corpus: Vec = (0..files) + .map(|_| { + let mut b = vec![0u8; mb * 1024 * 1024]; + fill_random(&mut b, &mut seed); + Bytes::from(b) + }) + .collect(); + + async fn write_all_entries(sink: W, corpus: &[Bytes]) { + let buf = tokio::io::BufWriter::with_capacity(256 * 1024, sink); + let mut zip = ZipFileWriter::with_tokio(buf); + for (i, data) in corpus.iter().enumerate() { + let entry = ZipEntryBuilder::new(format!("IMG_{i:04}.jpg").into(), Compression::Stored); + let mut w = zip.write_entry_stream(entry).await.unwrap(); + for c in data.chunks(64 * 1024) { + w.write_all(c).await.unwrap(); + } + w.close().await.unwrap(); + } + let mut compat = zip.close().await.unwrap(); + compat.close().await.unwrap(); + } + + // BEFORE: build the whole archive into a temp file, then "respond". + let t0 = Instant::now(); + let temp = tempfile::NamedTempFile::new().unwrap(); + let f = tokio::fs::File::create(temp.path()).await.unwrap(); + write_all_entries(f, &corpus).await; + // first byte = read back the first chunk + use tokio::io::AsyncReadExt; + let mut rf = tokio::fs::File::open(temp.path()).await.unwrap(); + let mut first = vec![0u8; 64 * 1024]; + rf.read_exact(&mut first).await.unwrap(); + let ttfb_before = t0.elapsed().as_secs_f64() * 1000.0; + let mut rest = Vec::new(); + rf.read_to_end(&mut rest).await.unwrap(); + let total_before = t0.elapsed().as_secs_f64() * 1000.0; + + // AFTER: duplex — first byte as soon as the first entry flushes. + let t0 = Instant::now(); + let (writer, reader) = tokio::io::duplex(256 * 1024); + let corpus2 = corpus.clone(); + let jh = tokio::spawn(async move { write_all_entries(writer, &corpus2).await }); + let mut rs = tokio_util::io::ReaderStream::new(reader); + let firstb = rs.next().await.unwrap().unwrap(); + std::hint::black_box(&firstb); + let ttfb_after = t0.elapsed().as_secs_f64() * 1000.0; + let mut drained = firstb.len(); + while let Some(c) = rs.next().await { + drained += c.unwrap().len(); + } + jh.await.unwrap(); + let total_after = t0.elapsed().as_secs_f64() * 1000.0; + + println!(" BEFORE temp-file : TTFB {ttfb_before:>8.1} ms total {total_before:>8.1} ms"); + println!( + " AFTER streaming : TTFB {ttfb_after:>8.1} ms total {total_after:>8.1} ms (TTFB {:.0}x, {} MiB drained)", + ttfb_before / ttfb_after.max(0.001), + drained / 1024 / 1024 + ); + println!(" (TTFB scales with archive size in BEFORE; constant in AFTER)"); +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + dotenvy::dotenv().ok(); + let url = env::var("DATABASE_URL").unwrap_or_default(); + let sections: Vec = env::var("BENCH_SECTIONS") + .unwrap_or_else(|_| "1,2,3,4,5".into()) + .split(',') + .filter_map(|x| x.trim().parse().ok()) + .collect(); + + let _ = median(vec![0.0]); // keep helper linked even if sections change + for s in sections { + match s { + 1 => section_range_cache(&url).await, + 2 => section_nc_chunk_gate().await, + 3 => section_delta_prefetch().await, + 4 => section_ingest_overlap(&url).await, + 5 => section_zip_stream().await, + _ => {} + } + } +} diff --git a/frontend/src/lib/api/endpoints/deltaUpload.hash.test.ts b/frontend/src/lib/api/endpoints/deltaUpload.hash.test.ts new file mode 100644 index 00000000..5d316ce7 --- /dev/null +++ b/frontend/src/lib/api/endpoints/deltaUpload.hash.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest'; +import { Worker } from 'node:worker_threads'; +import { createHash } from 'node:crypto'; +import { promises as fs } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** + * Benchmark gate for the worker-pool hashing in `resolveOwnedHashes`. + * + * The browser change moves per-file BLAKE3 hashing from a sequential + * main-thread WASM loop onto a small pool of Web Workers. This test measures + * the same architecture on this machine with node's worker_threads and a + * CPU-bound digest as the stand-in workload: N buffers hashed sequentially + * on one thread vs the same work fanned over a 3-lane pool. If the pool + * doesn't beat sequential wall-clock, the frontend change must be rolled + * back (it would be pure complexity). + */ +describe('worker-pool hashing (architecture gate)', () => { + it('a 3-lane pool beats sequential main-thread hashing on wall clock', async () => { + // Faithful to the browser shape: the main thread hands each worker a + // FILE REFERENCE (browser: the File handle; here: its path) and the + // worker does read + hash. The old shape reads + hashes every file + // on the main thread, serially. + const nFiles = 24; + const size = 4 * 1024 * 1024; + const dir = await fs.mkdtemp(join(tmpdir(), 'hashbench-')); + const paths: string[] = []; + for (let i = 0; i < nFiles; i++) { + const p = join(dir, `f${i}`); + const b = Buffer.alloc(size); + b.fill(i + 1); + await fs.writeFile(p, b); + paths.push(p); + } + + // Sequential (old): read + hash on the calling thread. + const t0 = performance.now(); + for (const p of paths) { + const b = await fs.readFile(p); + createHash('sha256').update(b).digest('hex'); + } + const seqMs = performance.now() - t0; + + // 3-lane pool (new): each worker reads + hashes its own files. + const lanes = 3; + const workerSrc = ` + const { parentPort } = require('node:worker_threads'); + const { createHash } = require('node:crypto'); + const { readFileSync } = require('node:fs'); + parentPort.on('message', (path) => { + const b = readFileSync(path); + parentPort.postMessage(createHash('sha256').update(b).digest('hex')); + }); + `; + const workers = Array.from({ length: lanes }, () => new Worker(workerSrc, { eval: true })); + let next = 0; + const t1 = performance.now(); + await Promise.all( + workers.map( + (w) => + new Promise((resolve, reject) => { + const feed = () => { + if (next >= paths.length) { + resolve(); + return; + } + const i = next++; + w.once('message', () => feed()); + w.once('error', reject); + w.postMessage(paths[i]); + }; + feed(); + }) + ) + ); + const poolMs = performance.now() - t1; + await Promise.all(workers.map((w) => w.terminate())); + await fs.rm(dir, { recursive: true, force: true }); + + // eslint-disable-next-line no-console + console.info( + `read+hash ${nFiles} x 4 MiB: sequential ${seqMs.toFixed(0)} ms vs 3-lane pool ${poolMs.toFixed(0)} ms (${(seqMs / poolMs).toFixed(1)}x)` + ); + expect(poolMs).toBeLessThan(seqMs); + }); +}); diff --git a/frontend/src/lib/api/endpoints/deltaUpload.ts b/frontend/src/lib/api/endpoints/deltaUpload.ts index c7af61cd..802e3cd3 100644 --- a/frontend/src/lib/api/endpoints/deltaUpload.ts +++ b/frontend/src/lib/api/endpoints/deltaUpload.ts @@ -176,6 +176,59 @@ export async function instantUploadOwned( return null; } +const HASH_WORKER_URL = '/workers/hashWorker.js'; +/** Parallel hashing lanes — enough to saturate small-file hashing without + * starving the upload workers of cores. */ +const HASH_POOL_SIZE = Math.min(4, Math.max(1, (navigator.hardwareConcurrency ?? 2) - 1)); + +/** + * BLAKE3-hash `files` on a bounded pool of dedicated workers (main thread + * stays free). A file whose worker errors is simply absent from the result — + * the caller uploads it the normal way. Falls back to the sequential inline + * hasher when `Worker` is unavailable. + */ +async function hashFilesPooled(files: File[]): Promise> { + if (typeof Worker === 'undefined') { + const out = new Map(); + for (const f of files) out.set(f, await blake3HexOfFile(f)); + return out; + } + const lanes = Math.min(HASH_POOL_SIZE, files.length); + const workers = Array.from( + { length: lanes }, + () => new Worker(HASH_WORKER_URL, { type: 'module' }) + ); + const out = new Map(); + let next = 0; + try { + await Promise.all( + workers.map( + (w) => + new Promise((resolve, reject) => { + const feed = () => { + if (next >= files.length) { + resolve(); + return; + } + const i = next++; + const file = files[i]; + w.onmessage = (ev: MessageEvent<{ id: number; hex?: string; error?: string }>) => { + if (ev.data.hex) out.set(file, ev.data.hex); + feed(); // per-file errors: skip the file, keep the lane + }; + w.onerror = (e) => reject(e); + w.postMessage({ id: i, file }); + }; + feed(); + }) + ) + ); + } finally { + for (const w of workers) w.terminate(); + } + return out; +} + /** * Resolve which of `files` the server already owns, with a SINGLE batch round * trip (the Dropbox-style "have you got these?" probe). Every file below the @@ -193,7 +246,13 @@ export async function resolveOwnedHashes(files: File[]): Promise(); try { - for (const f of inBand) hashByFile.set(f, await blake3HexOfFile(f)); + // Hash off the main thread on a small worker pool — the sequential + // main-thread WASM loop blocked the UI for the whole batch and + // delayed every upload lane behind the full hashing phase (measured + // in deltaUpload.hash.test.ts). Falls back to the inline loop when + // Workers are unavailable (some test environments). + const hashed = await hashFilesPooled(inBand); + for (const [f, h] of hashed) hashByFile.set(f, h); } catch { return new Map(); // WASM/hashing unavailable → skip instant uploads } diff --git a/frontend/static/workers/hashWorker.js b/frontend/static/workers/hashWorker.js new file mode 100644 index 00000000..af49009b --- /dev/null +++ b/frontend/static/workers/hashWorker.js @@ -0,0 +1,40 @@ +/** + * OxiCloud — whole-file BLAKE3 hashing worker. + * + * Computes the instant-upload ("does the server already own this?") hashes + * OFF the main thread. The previous shape hashed every small file of a + * batch drop sequentially on the main thread with synchronous WASM calls — + * seconds of UI jank for a large drop, all before the first upload lane + * even started (see collateral bench in deltaUpload.hash.test.ts). + * + * Protocol with the spawner (one worker handles many requests): + * in : { id: number, file: File } + * out : { id: number, hex: string } — success + * { id: number, error: string } — this file failed (caller + * falls back to plain upload) + */ + +const WASM_GLUE_URL = '/vendors/hash-wasm/oxicloud_hash_wasm.js'; + +let modPromise = null; +function load() { + if (!modPromise) { + modPromise = import(WASM_GLUE_URL).then(async (mod) => { + await mod.default(); + return mod; + }); + } + return modPromise; +} + +self.onmessage = async (ev) => { + const { id, file } = ev.data; + try { + const mod = await load(); + const bytes = new Uint8Array(await file.arrayBuffer()); + const hex = mod.blake3Hex(bytes); + self.postMessage({ id, hex }); + } catch (err) { + self.postMessage({ id, error: String(err) }); + } +}; diff --git a/src/application/ports/file_ports.rs b/src/application/ports/file_ports.rs index e2f6b064..fe9bac7a 100644 --- a/src/application/ports/file_ports.rs +++ b/src/application/ports/file_ports.rs @@ -111,6 +111,17 @@ pub enum OptimizedFileContent { Stream(Pin> + Send>>), } +/// Result of a cache-aware HTTP-Range read +/// (`FileRetrievalService::get_file_range_preloaded`). Same split as +/// [`OptimizedFileContent`]: handlers map each variant onto a response body. +pub enum RangeContent { + /// Zero-copy slice out of the RAM content cache (a `Bytes::slice` is a + /// refcount bump — no allocation, no I/O, no DB). + Bytes(Bytes), + /// Streaming range read from the blob store (cache miss / large file). + Stream(Box> + Send>), +} + /// Primary port for file retrieval operations pub trait FileRetrievalUseCase: Send + Sync + 'static { /// Gets a file by its ID (system/internal — no ownership check). diff --git a/src/application/services/delta_upload_service.rs b/src/application/services/delta_upload_service.rs index d21ca200..28927c9d 100644 --- a/src/application/services/delta_upload_service.rs +++ b/src/application/services/delta_upload_service.rs @@ -607,6 +607,12 @@ impl DeltaUploadService { Ok(DeltaDownloadOutcome::Ready(ordered)) } + /// Backend-recommended read-ahead depth for multi-chunk drains + /// (see `DedupService::read_prefetch`). + pub fn read_prefetch(&self) -> usize { + self.dedup.read_prefetch() + } + /// Stream one authorized chunk's bytes (entitlement was established by /// [`authorize_chunk_download_with_perms`]). pub async fn chunk_stream( diff --git a/src/application/services/file_retrieval_service.rs b/src/application/services/file_retrieval_service.rs index 7109739b..de03ac08 100644 --- a/src/application/services/file_retrieval_service.rs +++ b/src/application/services/file_retrieval_service.rs @@ -5,7 +5,9 @@ use std::sync::Arc; use crate::application::dtos::file_dto::FileDto; use crate::application::ports::authorization_ports::AuthorizationEngine; -use crate::application::ports::file_ports::{FileRetrievalUseCase, OptimizedFileContent}; +use crate::application::ports::file_ports::{ + FileRetrievalUseCase, OptimizedFileContent, RangeContent, +}; use crate::application::ports::resource_access_hook::ResourceAccessHook; use crate::application::ports::storage_ports::FileReadPort; use crate::common::errors::DomainError; @@ -284,6 +286,69 @@ impl FileRetrievalService { let files = self.file_read.get_files_by_ids(ids).await?; Ok(files.into_iter().map(FileDto::from).collect()) } + + /// Range read that first consults the RAM content cache (see + /// [`Self::get_file_range_preloaded`]). + pub async fn get_file_range_preloaded_with_perms( + &self, + dto: &FileDto, + caller_id: Uuid, + start: u64, + end: Option, + ) -> Result { + self.require_file(&dto.id, Permission::Read, caller_id) + .await?; + // Same throttled Recent recording as the streaming variant. + self.notify_file_accessed(caller_id, &dto.id); + self.get_file_range_preloaded(dto, start, end).await + } + + /// Range read for HTTP Range Requests, cache-aware. + /// + /// Media players and PDF viewers fetch these files *exclusively* through + /// Range requests (a `bytes=0-` probe, then seeks) — the plain streaming + /// path paid 1 PG round-trip (blob-hash resolve) + a chunk open/seek for + /// EVERY seek, even when the whole blob was already sitting in the moka + /// content cache as one contiguous `Bytes`. For sub-`CACHE_THRESHOLD` + /// files this now answers from the cache: `Bytes::slice` is a refcount + /// bump — zero copy, zero I/O, zero PG (benches/RANGE-CACHE.md). A miss + /// populates the cache via the same single-flight `get_or_load` Tier 1 + /// uses, so one probe warms every subsequent seek. `end` is exclusive + /// (callers pass `Some(last_byte + 1)`), matching the streaming variant. + pub async fn get_file_range_preloaded( + &self, + dto: &FileDto, + start: u64, + end: Option, + ) -> Result { + let cacheable = dto.size < CACHE_THRESHOLD && !dto.content_hash.is_empty(); + if cacheable && let Some(cache) = &self.content_cache { + let etag: Arc = format!("\"{}\"", dto.content_hash).into(); + let ct: Arc = dto.mime_type.clone(); + let file_read = Arc::clone(&self.file_read); + let id_owned = dto.id.clone(); + let cap = dto.size as usize; + let (bytes, _etag, _ct) = cache + .get_or_load(dto.content_hash.to_string(), etag, ct, async move { + debug!("💾 Range cache MISS: {} – loading from disk", id_owned); + Self::read_full(&file_read, &id_owned, cap).await + }) + .await?; + let len = bytes.len() as u64; + let s = start.min(len) as usize; + let e = end.unwrap_or(len).min(len) as usize; + if s <= e { + return Ok(RangeContent::Bytes(bytes.slice(s..e))); + } + // Degenerate range the validator should have rejected — fall + // through to the streaming path rather than panic on slice. + } + let stream = self + .file_read + .get_file_range_stream(&dto.id, start, end) + .await?; + Ok(RangeContent::Stream(stream)) + } } impl FileRetrievalUseCase for FileRetrievalService { diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 38e429d5..612e20d1 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -112,13 +112,33 @@ impl ChunkIngestOutcome { /// mid-stream — a client disconnect aborts the whole handler future — the /// guard spawns a rollback so pinned chunks don't leak references forever and /// written files become GC-collectible rows instead of invisible orphans. -struct IngestGuard { - pool: Arc, - backend: Arc, +/// Whether the ingest loop overlaps batch settling with source reading +/// (default on). `OXICLOUD_INGEST_OVERLAP=0` restores the old inline +/// behaviour — kept as a bench/ops escape hatch. +fn ingest_overlap_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| { + std::env::var("OXICLOUD_INGEST_OVERLAP").map_or(true, |v| v != "0" && v != "false") + }) +} + +/// Compensation ledger of one ingest session. Shared (`Arc`) +/// between the ingest loop and the overlapped batch-settle task: the settler +/// holds the lock for the whole batch and records progressively, so a +/// rollback (explicit or Drop-spawned) that acquires the lock is guaranteed +/// to observe every pin/write the in-flight settle made. +#[derive(Default)] +struct IngestState { /// Pre-existing chunks whose ref_count this session bumped (distinct). pinned: Vec, /// Chunks written to the backend but not yet registered: (hash, size). written: Vec<(String, i64)>, +} + +struct IngestGuard { + pool: Arc, + backend: Arc, + state: Arc>, armed: bool, } @@ -127,8 +147,7 @@ impl IngestGuard { Self { pool, backend, - pinned: Vec::new(), - written: Vec::new(), + state: Arc::new(tokio::sync::Mutex::new(IngestState::default())), armed: true, } } @@ -143,8 +162,15 @@ impl IngestGuard { /// spawned Drop path). async fn rollback(mut self) { self.armed = false; - let pinned = std::mem::take(&mut self.pinned); - let written = std::mem::take(&mut self.written); + // Lock acquisition serializes after any in-flight batch settle, so + // its pins/writes are visible here. + let (pinned, written) = { + let mut st = self.state.lock().await; + ( + std::mem::take(&mut st.pinned), + std::mem::take(&mut st.written), + ) + }; Self::run_rollback(self.pool.clone(), self.backend.clone(), pinned, written).await; } @@ -211,24 +237,33 @@ impl IngestGuard { impl Drop for IngestGuard { fn drop(&mut self) { - if !self.armed || (self.pinned.is_empty() && self.written.is_empty()) { + if !self.armed { return; } - let pinned = std::mem::take(&mut self.pinned); - let written = std::mem::take(&mut self.written); + // The rollback task locks the shared state first, so it naturally + // waits out an in-flight batch settle and observes its recordings. + let state = self.state.clone(); match tokio::runtime::Handle::try_current() { Ok(handle) => { let pool = self.pool.clone(); let backend = self.backend.clone(); handle.spawn(async move { + let (pinned, written) = { + let mut st = state.lock().await; + ( + std::mem::take(&mut st.pinned), + std::mem::take(&mut st.written), + ) + }; + if pinned.is_empty() && written.is_empty() { + return; + } Self::run_rollback(pool, backend, pinned, written).await; }); } Err(_) => tracing::warn!( - "Ingest guard dropped outside a runtime: {} pins / {} written chunks \ + "Ingest guard dropped outside a runtime: any pins / written chunks \ stay leaked until the next GC sweep", - pinned.len(), - written.len() ), } } @@ -628,6 +663,13 @@ impl DedupService { .map_err(|e| DomainError::internal_error("Dedup", format!("chunk_sizes query: {e}"))) } + /// Read-ahead depth the backend recommends for multi-chunk drains + /// (1 local, 8 for request-latency-bound object stores) — see + /// `BlobStorageBackend::read_prefetch` and benches/BLOB-PREFETCH.md. + pub fn read_prefetch(&self) -> usize { + self.backend.read_prefetch() + } + /// Stream one chunk's raw bytes from the backend. The caller is /// responsible for entitlement (see [`claimable_chunks`]). pub async fn chunk_stream( @@ -876,8 +918,28 @@ impl DedupService { let mut hasher = blake3::Hasher::new(); let mut head: Vec = Vec::with_capacity(sniff_len.min(16 * 1024)); - for (hash, declared_size) in chunks { - let mut stream = self.backend.get_blob_stream(hash).await?; + // Overlap the NEXT chunk's open with the current chunk's hash+drain + // — the same `buffered(read_prefetch)` combinator as the download + // path (benches/BLOB-PREFETCH.md measured +7-12 % on local disk; + // request-latency-bound object stores gain far more). Hashing stays + // strictly in manifest order: `buffered` yields in input order. + let prefetch = self.backend.read_prefetch().max(1); + let backend = self.backend.clone(); + let mut opened = futures::stream::iter(chunks.iter().cloned()) + .map(move |(hash, declared_size)| { + let backend = backend.clone(); + async move { + backend + .get_blob_stream(&hash) + .await + .map(|s| (hash, declared_size, s)) + } + }) + .buffered(prefetch); + + while let Some(next) = opened.next().await { + let (hash, declared_size, mut stream) = next?; + let (hash, declared_size) = (&hash, &declared_size); let mut actual: u64 = 0; while let Some(part) = stream.next().await { let part = part.map_err(|e| { @@ -954,7 +1016,7 @@ impl DedupService { where S: Stream> + Send, { - let mut guard = IngestGuard::new(self.pool.clone(), self.backend.clone()); + let guard = IngestGuard::new(self.pool.clone(), self.backend.clone()); let reader = StreamReader::new(Box::pin(source)); let mut chunker = fastcdc::v2020::AsyncStreamCDC::new( @@ -973,11 +1035,35 @@ impl DedupService { let mut session_seen: HashSet = HashSet::new(); let mut pending: Vec<(String, Bytes)> = Vec::new(); let mut pending_bytes: usize = 0; + // Depth-1 settle pipeline: batch N settles on a spawned task while + // the loop keeps reading/chunking/hashing batch N+1 from the source + // — the inline shape froze the reader (and the client's socket) for + // every settle (benches/INGEST-OVERLAP.md). The task records into + // the guard's shared state under its lock, so rollback stays exact + // even if this future is dropped mid-settle. + let mut in_flight: Option>> = None; + + /// Await the previous batch's settle, mapping panics/aborts to a + /// domain error so both are compensated identically. + async fn join_settle( + handle: tokio::task::JoinHandle>, + ) -> Result<(), DomainError> { + match handle.await { + Ok(res) => res, + Err(e) => Err(DomainError::internal_error( + "Dedup", + format!("Chunk settle task failed: {e}"), + )), + } + } while let Some(item) = chunk_stream.next().await { let chunk = match item { Ok(chunk) => chunk, Err(e) => { + if let Some(handle) = in_flight.take() { + let _ = join_settle(handle).await; + } guard.rollback().await; return Err(DomainError::internal_error( "Dedup", @@ -1000,7 +1086,26 @@ impl DedupService { pending.push((hash, Bytes::from(data))); if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES { - if let Err(e) = self.flush_pending(&mut guard, &mut pending).await { + if let Some(handle) = in_flight.take() + && let Err(e) = join_settle(handle).await + { + guard.rollback().await; + return Err(e); + } + let batch = std::mem::take(&mut pending); + let handle = tokio::spawn(Self::settle_batch( + self.pool.clone(), + self.backend.clone(), + guard.state.clone(), + batch, + )); + // Bench/ops escape hatch: OXICLOUD_INGEST_OVERLAP=0 + // reproduces the old inline-settle behaviour (await the + // batch before reading on) — used by + // benches/INGEST-OVERLAP.md for an in-binary A/B. + if ingest_overlap_enabled() { + in_flight = Some(handle); + } else if let Err(e) = join_settle(handle).await { guard.rollback().await; return Err(e); } @@ -1009,7 +1114,20 @@ impl DedupService { } } - if let Err(e) = self.flush_pending(&mut guard, &mut pending).await { + if let Some(handle) = in_flight.take() + && let Err(e) = join_settle(handle).await + { + guard.rollback().await; + return Err(e); + } + if let Err(e) = Self::settle_batch( + self.pool.clone(), + self.backend.clone(), + guard.state.clone(), + std::mem::take(&mut pending), + ) + .await + { guard.rollback().await; return Err(e); } @@ -1018,10 +1136,15 @@ impl DedupService { // One batched fsync sweep (no-op for remote backends, durable on // PUT), then one batched INSERT. A crash before the INSERT leaves // only unreferenced files; never a row pointing at unsynced bytes. - if !guard.written.is_empty() { - let new_hashes: Vec = guard.written.iter().map(|(h, _)| h.clone()).collect(); - let new_sizes: Vec = guard.written.iter().map(|(_, s)| *s).collect(); - + // No settle is in flight past this point — the lock is uncontended. + let (new_hashes, new_sizes): (Vec, Vec) = { + let st = guard.state.lock().await; + ( + st.written.iter().map(|(h, _)| h.clone()).collect(), + st.written.iter().map(|(_, s)| *s).collect(), + ) + }; + if !new_hashes.is_empty() { if let Err(e) = self.backend.sync_blobs(&new_hashes).await { guard.rollback().await; return Err(e); @@ -1047,7 +1170,7 @@ impl DedupService { } } - let newly_written = guard.written.len(); + let newly_written = new_hashes.len(); guard.disarm(); Ok(ChunkIngestOutcome { @@ -1061,18 +1184,23 @@ impl DedupService { /// Settle one batch of distinct in-RAM chunks against PG + the backend. /// - /// Successfully pinned hashes and written chunks are recorded on the - /// guard as they happen, so a failure mid-batch leaves nothing - /// untracked for rollback. - async fn flush_pending( - &self, - guard: &mut IngestGuard, - pending: &mut Vec<(String, Bytes)>, + /// Static (no `&self`) so the ingest loop can run it on a spawned task + /// and keep consuming the source stream while the batch settles — the + /// inline shape stalled the reader for the whole settle every 8 MiB + /// (benches/INGEST-OVERLAP.md). The shared-state lock is held for the + /// entire batch: pinned hashes and written chunks are recorded + /// progressively under it, so a failure (or a rollback racing this + /// settle) leaves nothing untracked. + async fn settle_batch( + pool: Arc, + backend: Arc, + state: Arc>, + batch: Vec<(String, Bytes)>, ) -> Result<(), DomainError> { - if pending.is_empty() { + if batch.is_empty() { return Ok(()); } - let batch = std::mem::take(pending); + let mut guard = state.lock().await; let hashes: Vec = batch.iter().map(|(h, _)| h.clone()).collect(); // Pin-or-classify in one statement: rows that exist take this @@ -1084,7 +1212,7 @@ impl DedupService { RETURNING hash", ) .bind(&hashes) - .fetch_all(self.pool.as_ref()) + .fetch_all(pool.as_ref()) .await .map_err(|e| { DomainError::internal_error("Dedup", format!("Failed to pin existing chunks: {e}")) @@ -1106,7 +1234,6 @@ impl DedupService { // Unsynced writes — durability comes from the single end-of-stream // sweep, before any PG row references these chunks. - let backend = self.backend.clone(); let results: Vec> = stream::iter(to_write) .map(|(hash, data)| { let backend = backend.clone(); diff --git a/src/infrastructure/services/nextcloud_chunked_upload_service.rs b/src/infrastructure/services/nextcloud_chunked_upload_service.rs index f305fa19..1ddd7a1c 100644 --- a/src/infrastructure/services/nextcloud_chunked_upload_service.rs +++ b/src/infrastructure/services/nextcloud_chunked_upload_service.rs @@ -1,24 +1,84 @@ use std::path::PathBuf; +use std::time::Duration; use tokio::fs; use crate::common::errors::{DomainError, Result}; +/// In-RAM running byte counter per upload session (`user/upload_id` → +/// bytes accepted so far). The per-chunk quota gate used to recompute +/// this by listing the whole session directory and stat-ing every chunk +/// on EVERY chunk PUT — O(k) stats for chunk k, O(N²/2) over an upload +/// (~500k stats for a 10 GB / 1000-chunk upload). The counter makes the +/// gate O(1); a cache miss (process restart, eviction) lazily rebuilds +/// from the directory listing, so crash-correctness is unchanged +/// (benches/NC-CHUNK-GATE.md). Sessions are forgotten on cleanup; the +/// TTL reaps counters for sessions the client abandoned. +fn build_session_bytes_cache() -> moka::sync::Cache { + moka::sync::Cache::builder() + .max_capacity(100_000) + .time_to_idle(Duration::from_secs(24 * 3600)) + .build() +} + #[derive(Clone)] pub struct NextcloudChunkedUploadService { pub base_dir: PathBuf, + /// See [`build_session_bytes_cache`]. Cloning the service shares the + /// counter (moka `Cache` clones are handles to the same store). + session_bytes: moka::sync::Cache, } impl NextcloudChunkedUploadService { pub fn new(base_dir: PathBuf) -> Self { - Self { base_dir } + Self { + base_dir, + session_bytes: build_session_bytes_cache(), + } } pub fn new_stub() -> Self { Self { base_dir: PathBuf::from("./storage/.uploads/nextcloud"), + session_bytes: build_session_bytes_cache(), } } + fn bytes_key(user: &str, upload_id: &str) -> String { + format!("{user}/{upload_id}") + } + + /// Session bytes accepted so far, if the counter is warm. + /// `None` = rebuild from the directory listing and call + /// [`Self::set_session_bytes`]. + pub fn cached_session_bytes(&self, user: &str, upload_id: &str) -> Option { + self.session_bytes.get(&Self::bytes_key(user, upload_id)) + } + + /// Seed / overwrite the session counter (post-rebuild or on MKCOL). + pub fn set_session_bytes(&self, user: &str, upload_id: &str, bytes: u64) { + self.session_bytes + .insert(Self::bytes_key(user, upload_id), bytes); + } + + /// Add an accepted chunk's bytes to the counter (no-op when cold — + /// the next gate rebuilds from disk). Two racing PUTs on one session + /// could drop an increment; the counter is a gate hint, and the + /// MOVE-time quota check stays authoritative. + pub fn bump_session_bytes(&self, user: &str, upload_id: &str, delta: u64) { + let key = Self::bytes_key(user, upload_id); + if let Some(current) = self.session_bytes.get(&key) { + self.session_bytes + .insert(key, current.saturating_add(delta)); + } + } + + /// Drop the counter (session cleanup, or a chunk overwrite made the + /// running total untrustworthy — rebuilt lazily on next use). + pub fn forget_session_bytes(&self, user: &str, upload_id: &str) { + self.session_bytes + .invalidate(&Self::bytes_key(user, upload_id)); + } + /// Validate that a path component contains no traversal characters. fn validate_path_component(name: &str, label: &str) -> Result<()> { if name.is_empty() @@ -48,6 +108,7 @@ impl NextcloudChunkedUploadService { fs::create_dir_all(&session_dir) .await .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; + self.set_session_bytes(user, upload_id, 0); Ok(()) } @@ -97,9 +158,17 @@ impl NextcloudChunkedUploadService { data: &[u8], ) -> Result<()> { let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?; + let overwrite = fs::metadata(&chunk_path).await.is_ok(); fs::write(&chunk_path, data) .await - .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string())) + .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; + if overwrite { + // Retried chunk — running total is stale; rebuild lazily. + self.forget_session_bytes(user, upload_id); + } else { + self.bump_session_bytes(user, upload_id, data.len() as u64); + } + Ok(()) } /// List the session's chunk files in assembly (numeric) order. @@ -146,6 +215,7 @@ impl NextcloudChunkedUploadService { .await .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; } + self.forget_session_bytes(user, upload_id); Ok(()) } diff --git a/src/infrastructure/services/zip_service.rs b/src/infrastructure/services/zip_service.rs index b84087d5..d5ef1fcf 100644 --- a/src/infrastructure/services/zip_service.rs +++ b/src/infrastructure/services/zip_service.rs @@ -44,8 +44,9 @@ impl From for DomainError { } } -/// Type alias for the fully-async ZIP writer backed by a buffered tokio file. -type AsyncZipWriter = ZipFileWriter>>; +/// Fully-async ZIP writer over any buffered tokio sink (temp file for the +/// legacy path, one half of a `tokio::io::duplex` for the streaming path). +type AsyncZipWriter = ZipFileWriter>>; /// One planned archive entry, in final ZIP order. enum ZipPlanEntry { @@ -119,6 +120,110 @@ impl ZipService { folder_id: &str, folder_name: &str, ) -> Result { + let plan = self.plan_archive(folder_id, folder_name).await?; + + // ── Open the temp file + ZIP writer ────────────────────────────── + let temp = NamedTempFile::new().map_err(ZipError::IoError)?; + let tokio_file = tokio::fs::File::create(temp.path()) + .await + .map_err(ZipError::IoError)?; + + let (tx, mut rx) = tokio::sync::mpsc::channel::(PREFETCH_BUFFER_CHUNKS); + let _prefetcher = tokio::spawn(Self::prefetch_files( + self.file_service.clone(), + Self::planned_file_ids(&plan), + tx, + )); + Self::write_archive(tokio_file, &plan, &mut rx).await?; + + Ok(temp) + } + + /// Streaming variant: the archive bytes are produced on a spawned task + /// and yielded as they are written — the client's first byte arrives + /// after the first entry starts, not after the whole archive has been + /// built (the temp-file variant's time-to-first-byte grows with folder + /// size; benches/ZIP-STREAM.md). The plan phase still runs inline so + /// planning errors surface as proper HTTP errors; a blob-read error + /// mid-archive can only truncate the stream (no central directory → + /// clients detect the corrupt archive), which is the standard tradeoff + /// for streamed ZIPs. + pub async fn create_folder_zip_stream( + &self, + folder_id: &str, + folder_name: &str, + ) -> Result> + Send + use<>> { + let plan = self.plan_archive(folder_id, folder_name).await?; + + let (writer, reader) = tokio::io::duplex(256 * 1024); + let (tx, mut rx) = tokio::sync::mpsc::channel::(PREFETCH_BUFFER_CHUNKS); + let _prefetcher = tokio::spawn(Self::prefetch_files( + self.file_service.clone(), + Self::planned_file_ids(&plan), + tx, + )); + tokio::spawn(async move { + if let Err(e) = Self::write_archive(writer, &plan, &mut rx).await { + // Dropping the writer EOFs the reader early — the truncated + // archive has no central directory, so clients flag it. + warn!("Streaming ZIP aborted mid-archive: {e}"); + } + }); + + Ok(tokio_util::io::ReaderStream::new(reader)) + } + + /// File ids of the plan, in archive order (the prefetcher's read list). + fn planned_file_ids(plan: &[ZipPlanEntry]) -> Vec { + plan.iter() + .filter_map(|entry| match entry { + ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()), + ZipPlanEntry::Dir(_) => None, + }) + .collect() + } + + /// Write every planned entry through a buffered ZIP writer over `sink`, + /// then finalize (central directory + flush). Shared by the temp-file + /// and streaming variants. + async fn write_archive( + sink: W, + plan: &[ZipPlanEntry], + rx: &mut tokio::sync::mpsc::Receiver, + ) -> Result<()> { + let buf_writer = BufWriter::with_capacity(256 * 1024, sink); + let mut zip = ZipFileWriter::with_tokio(buf_writer); + + for entry in plan { + match entry { + ZipPlanEntry::Dir(zip_dir) => { + let dir_entry = + ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored); + match zip.write_entry_whole(dir_entry, &[]).await { + Ok(()) => debug!("Folder added to ZIP: {}", zip_dir), + Err(e) => { + warn!("Could not add folder entry (may already exist): {}", e); + } + } + } + ZipPlanEntry::File { + zip_path, + compression, + .. + } => { + Self::write_prefetched_file(&mut zip, zip_path, *compression, rx).await?; + } + } + } + + let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?; + compat_writer.close().await.map_err(ZipError::IoError)?; + Ok(()) + } + + /// Resolve the folder, fetch its subtree (2 bulk queries) and lay out + /// the archive entries in final ZIP order. + async fn plan_archive(&self, folder_id: &str, folder_name: &str) -> Result> { info!( "Creating ZIP for folder: {} (ID: {})", folder_name, folder_id @@ -200,61 +305,7 @@ impl ZipService { } } - // ── 5. Open the temp file + ZIP writer ─────────────────────────── - let temp = NamedTempFile::new().map_err(ZipError::IoError)?; - let tokio_file = tokio::fs::File::create(temp.path()) - .await - .map_err(ZipError::IoError)?; - let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file); - let mut zip = ZipFileWriter::with_tokio(buf_writer); - - // ── 6. Write entries: 2-stage pipeline ─────────────────────────── - // The prefetch task reads blob streams for the planned files, in - // order, ahead of the writer — the next file's blob-store latency - // overlaps the current file's deflate. If the writer bails out, - // dropping the receiver makes the prefetcher's next send fail and - // it stops on its own. - let file_ids: Vec = plan - .iter() - .filter_map(|entry| match entry { - ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()), - ZipPlanEntry::Dir(_) => None, - }) - .collect(); - let (tx, mut rx) = tokio::sync::mpsc::channel::(PREFETCH_BUFFER_CHUNKS); - let _prefetcher = tokio::spawn(Self::prefetch_files( - self.file_service.clone(), - file_ids, - tx, - )); - - for entry in &plan { - match entry { - ZipPlanEntry::Dir(zip_dir) => { - let dir_entry = - ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored); - match zip.write_entry_whole(dir_entry, &[]).await { - Ok(()) => debug!("Folder added to ZIP: {}", zip_dir), - Err(e) => { - warn!("Could not add folder entry (may already exist): {}", e); - } - } - } - ZipPlanEntry::File { - zip_path, - compression, - .. - } => { - Self::write_prefetched_file(&mut zip, zip_path, *compression, &mut rx).await?; - } - } - } - - // ── 7. Finalize ────────────────────────────────────────────────── - let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?; - compat_writer.close().await.map_err(ZipError::IoError)?; - - Ok(temp) + Ok(plan) } /// Prefetch stage: streams each planned file's content from the blob @@ -302,8 +353,8 @@ impl ZipService { /// (`Stored` for already-compressed media, `Deflate` otherwise — see /// `entry_compression`). Peak memory stays bounded by the channel, /// independent of individual file sizes. - async fn write_prefetched_file( - zip: &mut AsyncZipWriter, + async fn write_prefetched_file( + zip: &mut AsyncZipWriter, zip_path: &str, compression: Compression, rx: &mut tokio::sync::mpsc::Receiver, diff --git a/src/interfaces/api/handlers/delta_upload_handler.rs b/src/interfaces/api/handlers/delta_upload_handler.rs index 6912d37f..7d2fa471 100644 --- a/src/interfaces/api/handlers/delta_upload_handler.rs +++ b/src/interfaces/api/handlers/delta_upload_handler.rs @@ -19,7 +19,7 @@ use axum::{ response::{IntoResponse, Response}, }; use bytes::{Buf, Bytes, BytesMut}; -use futures::Stream; +use futures::{Stream, TryStreamExt}; use std::sync::Arc; use tokio_stream::StreamExt; @@ -343,17 +343,36 @@ pub async fn delta_download_chunks( // Stream the frames: 4-byte length headers come from the (entitled) // index sizes; bytes stream straight from the blob backend. Peak RAM - // is one backend read frame, independent of batch size. + // is bounded by `read_prefetch` open streams (their first frame), + // independent of batch size. + // + // `buffered(read_prefetch)` overlaps the NEXT chunk's open with the + // current chunk's drain — the same combinator/tuning as the main CDC + // download path (benches/BLOB-PREFETCH.md). The old per-chunk await + // paid every open's full round-trip serially: on an object-store + // backend a 64-chunk batch at ~30 ms first-byte cost ~1.9 s of pure + // latency. Frames still arrive strictly in request order. + let prefetch = service.read_prefetch().max(1); + let svc = service.clone(); + // `futures::StreamExt` spelled out — this handler imports + // `tokio_stream::StreamExt`, whose `map` adapter lacks `buffered`. + let opened = futures::StreamExt::map(futures::stream::iter(ordered), move |(hash, size)| { + let svc = svc.clone(); + async move { + let chunk = svc + .chunk_stream(&hash) + .await + .map_err(std::io::Error::other)?; + let header = futures::stream::once(async move { + Ok::(Bytes::copy_from_slice(&(size as u32).to_be_bytes())) + }); + Ok::<_, std::io::Error>(futures::StreamExt::chain(header, chunk)) + } + }); let body_stream: std::pin::Pin> + Send>> = - Box::pin(async_stream::try_stream! { - for (hash, size) in ordered { - yield Bytes::copy_from_slice(&(size as u32).to_be_bytes()); - let mut chunk = service.chunk_stream(&hash).await.map_err(std::io::Error::other)?; - while let Some(part) = chunk.next().await { - yield part?; - } - } - }); + Box::pin(TryStreamExt::try_flatten(futures::StreamExt::buffered( + opened, prefetch, + ))); Ok(Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/octet-stream") diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index 7347f415..f1095780 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -12,7 +12,7 @@ use std::collections::HashMap; use utoipa::ToSchema; use crate::application::ports::file_ports::{ - FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, + FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, RangeContent, }; use crate::application::ports::storage_ports::{FileReadPort, StorageUsagePort}; use crate::application::ports::thumbnail_ports::ThumbnailPort; @@ -713,10 +713,19 @@ impl FileHandler { Self::content_disposition(&file_dto.name, &file_dto.mime_type, ¶ms); match retrieval - .get_file_range_stream_with_perms(&id, auth_user.id, start, Some(end + 1)) + .get_file_range_preloaded_with_perms( + &file_dto, + auth_user.id, + start, + Some(end + 1), + ) .await { - Ok(stream) => { + Ok(content) => { + let body = match content { + RangeContent::Bytes(b) => Body::from(b), + RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)), + }; return Response::builder() .status(StatusCode::PARTIAL_CONTENT) .header(header::CONTENT_TYPE, &*file_dto.mime_type) @@ -732,7 +741,7 @@ impl FileHandler { header::CACHE_CONTROL, "private, max-age=3600, must-revalidate", ) - .body(Body::from_stream(Box::into_pin(stream))) + .body(body) .unwrap() .into_response(); } diff --git a/src/interfaces/api/handlers/folder_handler.rs b/src/interfaces/api/handlers/folder_handler.rs index d2957ded..084d0048 100644 --- a/src/interfaces/api/handlers/folder_handler.rs +++ b/src/interfaces/api/handlers/folder_handler.rs @@ -6,7 +6,6 @@ use axum::{ }; use std::collections::HashMap; use std::sync::Arc; -use tokio_util::io::ReaderStream; use crate::application::dtos::display_helpers::{ category_for, format_file_size, icon_class_for, icon_special_class_for, @@ -238,53 +237,28 @@ impl FolderHandler { } }; - // Create the ZIP archive (written to a temp file, O(1) RAM) - match zip_service.create_folder_zip(&id, &folder.name).await { - Ok(temp_file) => { - // Get the file size for Content-Length - let file_size = match temp_file.as_file().metadata() { - Ok(m) => m.len(), - Err(e) => { - tracing::error!("Error reading temp file metadata: {}", e); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(serde_json::json!({ - "error": "Error creating ZIP file" - })), - ) - .into_response(); - } - }; - - tracing::info!("ZIP file created successfully, size: {} bytes", file_size); - - // Split the NamedTempFile into the already-open std File - // and the TempPath (auto-deletes on drop). This reuses - // the existing fd instead of opening a second one. - let (std_file, temp_path) = temp_file.into_parts(); - let tokio_file = tokio::fs::File::from_std(std_file); - - // Stream the file to the client in chunks - let stream = ReaderStream::new(tokio_file); + // Stream the archive as it is built — the first byte reaches + // the client after the first entry, not after the whole ZIP + // exists on disk (benches/ZIP-STREAM.md). No Content-Length: + // the final size isn't known up front (chunked encoding). + match zip_service + .create_folder_zip_stream(&id, &folder.name) + .await + { + Ok(stream) => { let body = axum::body::Body::from_stream(stream); // Setup headers for download let filename = format!("{}.zip", folder.name); let content_disposition = format!("attachment; filename=\"{}\"", filename); - let mut response = Response::builder() + Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/zip") .header(header::CONTENT_DISPOSITION, content_disposition) - .header(header::CONTENT_LENGTH, file_size) .body(body) - .unwrap(); - - // Keep TempPath alive in the response extensions so the - // file is only deleted AFTER the body stream finishes. - response.extensions_mut().insert(Arc::new(temp_path)); - - response.into_response() + .unwrap() + .into_response() } Err(err) => { tracing::error!("Error creating ZIP file: {}", err); diff --git a/src/interfaces/api/handlers/share_handler.rs b/src/interfaces/api/handlers/share_handler.rs index 2bda63b9..c3801e84 100644 --- a/src/interfaces/api/handlers/share_handler.rs +++ b/src/interfaces/api/handlers/share_handler.rs @@ -13,6 +13,7 @@ use serde::Deserialize; use serde_json::json; use utoipa::ToSchema; +use crate::application::ports::file_ports::RangeContent; use crate::application::services::share_browse_service::ZipTarget; use crate::application::services::share_service::ShareService; use crate::infrastructure::services::share_unlock_cookie; @@ -30,7 +31,6 @@ use crate::{ interfaces::errors::AppError, interfaces::middleware::auth::AuthUser, }; -use tokio_util::io::ReaderStream; fn unlock_jwt_from_headers(headers: &HeaderMap, share_token: &str) -> Option { headers @@ -438,10 +438,14 @@ async fn serve_share_file( let length = end - start + 1; match retrieval - .get_file_range_stream(file_id, start, Some(end + 1)) + .get_file_range_preloaded(&file_dto, start, Some(end + 1)) .await { - Ok(stream) => { + Ok(content) => { + let body = match content { + RangeContent::Bytes(b) => Body::from(b), + RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)), + }; return Response::builder() .status(StatusCode::PARTIAL_CONTENT) .header(header::CONTENT_TYPE, &*mime) @@ -458,7 +462,7 @@ async fn serve_share_file( "private, max-age=3600, must-revalidate", ) .header(header::VARY, "Cookie, Range") - .body(Body::from_stream(Box::into_pin(stream))) + .body(body) .unwrap() .into_response(); } @@ -730,30 +734,19 @@ async fn serve_share_zip( Err(err) => return share_browse_error_response(err), }; - let temp_file = match zip_service - .create_folder_zip(&target.folder_id, &target.display_name) + // Streamed archive: first byte after the first entry, not after the + // whole ZIP is built (benches/ZIP-STREAM.md). No Content-Length. + let stream = match zip_service + .create_folder_zip_stream(&target.folder_id, &target.display_name) .await { - Ok(f) => f, + Ok(s) => s, Err(err) => { tracing::error!("share zip: create_folder_zip failed: {}", err); return AppError::internal_error(format!("ZIP creation failed: {}", err)) .into_response(); } }; - - let file_size = match temp_file.as_file().metadata() { - Ok(m) => m.len(), - Err(e) => { - tracing::error!("share zip: temp metadata failed: {}", e); - return AppError::internal_error("ZIP creation failed").into_response(); - } - }; - - // Reuse the existing fd: split off the std::File and the TempPath. - let (std_file, temp_path) = temp_file.into_parts(); - let tokio_file = tokio::fs::File::from_std(std_file); - let stream = ReaderStream::new(tokio_file); let body = Body::from_stream(stream); let disposition = build_content_disposition( @@ -762,17 +755,12 @@ async fn serve_share_zip( false, ); - let mut response = Response::builder() + Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/zip") .header(header::CONTENT_DISPOSITION, disposition) - .header(header::CONTENT_LENGTH, file_size) .header(header::CACHE_CONTROL, "private, no-store") .header(header::VARY, "Cookie") .body(body) - .unwrap(); - - // Keep TempPath alive until the body finishes streaming. - response.extensions_mut().insert(Arc::new(temp_path)); - response + .unwrap() } diff --git a/src/interfaces/nextcloud/uploads_handler.rs b/src/interfaces/nextcloud/uploads_handler.rs index 85089da8..194b9c8a 100644 --- a/src/interfaces/nextcloud/uploads_handler.rs +++ b/src/interfaces/nextcloud/uploads_handler.rs @@ -74,6 +74,14 @@ async fn session_bytes_so_far( username: &str, upload_id: &str, ) -> Result { + // Warm path: O(1) in-RAM counter maintained by the PUT handler and + // the service (seeded on MKCOL, dropped on cleanup/overwrite). The + // directory walk below only runs cold (restart / eviction) — the old + // shape ran it on EVERY chunk PUT: O(k) stats for chunk k, O(N²/2) + // over the upload (benches/NC-CHUNK-GATE.md). + if let Some(bytes) = nc.chunked_uploads.cached_session_bytes(username, upload_id) { + return Ok(bytes); + } let listing = nc .chunked_uploads .list_chunks(username, upload_id) @@ -85,7 +93,10 @@ async fn session_bytes_so_far( // chunk after MKCOL (race-tolerant). return Ok(0); }; - Ok(listing.chunks.iter().map(|c| c.size).sum()) + let total = listing.chunks.iter().map(|c| c.size).sum(); + nc.chunked_uploads + .set_session_bytes(username, upload_id, total); + Ok(total) } /// Dispatch Nextcloud chunked upload WebDAV requests. @@ -314,11 +325,21 @@ async fn handle_put_chunk( .map_err(|e| AppError::bad_request(format!("Invalid chunk path: {}", e)))?; let max_chunk = state.core.config.storage.chunk_max_bytes; + // A re-PUT of an existing chunk (client retry) makes the running + // session counter stale — drop it so the next gate rebuilds from disk. + let overwrite = tokio::fs::metadata(&chunk_path).await.is_ok(); // No client-side integrity contract on the NC chunked surface — the // NC desktop client validates the assembled-file ETag against the // server-side `oc:checksums` after MOVE. So we skip per-chunk // hashing here (peak heap stays at ~one HTTP frame). - stream_body_to_path(req.into_body(), &chunk_path, max_chunk, None).await?; + let streamed = stream_body_to_path(req.into_body(), &chunk_path, max_chunk, None).await?; + if overwrite { + nc.chunked_uploads + .forget_session_bytes(&user.username, upload_id); + } else { + nc.chunked_uploads + .bump_session_bytes(&user.username, upload_id, streamed.bytes_written); + } Ok(Response::builder() .status(StatusCode::CREATED) diff --git a/src/interfaces/range_requests.rs b/src/interfaces/range_requests.rs index b29296f8..66ea36fa 100644 --- a/src/interfaces/range_requests.rs +++ b/src/interfaces/range_requests.rs @@ -13,7 +13,7 @@ use http_range_header::parse_range_header; use std::sync::Arc; use crate::application::dtos::file_dto::FileDto; -use crate::application::ports::file_ports::FileRetrievalUseCase; +use crate::application::ports::file_ports::RangeContent; use crate::application::services::file_retrieval_service::FileRetrievalService; /// `If-None-Match` short-circuit: returns a `304 Not Modified` response @@ -71,24 +71,32 @@ pub async fn range_response( let end = *range.end(); let range_length = end - start + 1; + // Cache-aware: sub-threshold files already in the RAM content cache are + // answered with a zero-copy Bytes slice — no PG, no disk (benches/RANGE-CACHE.md). match retrieval - .get_file_range_stream(&file.id, start, Some(end + 1)) + .get_file_range_preloaded(file, start, Some(end + 1)) .await { - Ok(stream) => Some( - Response::builder() - .status(StatusCode::PARTIAL_CONTENT) - .header(header::CONTENT_TYPE, &*file.mime_type) - .header(header::CONTENT_LENGTH, range_length) - .header( - header::CONTENT_RANGE, - format!("bytes {}-{}/{}", start, end, file.size), - ) - .header(header::ACCEPT_RANGES, "bytes") - .header(header::ETAG, etag) - .body(Body::from_stream(Box::into_pin(stream))) - .unwrap(), - ), + Ok(content) => { + let body = match content { + RangeContent::Bytes(b) => Body::from(b), + RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)), + }; + Some( + Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header(header::CONTENT_TYPE, &*file.mime_type) + .header(header::CONTENT_LENGTH, range_length) + .header( + header::CONTENT_RANGE, + format!("bytes {}-{}/{}", start, end, file.size), + ) + .header(header::ACCEPT_RANGES, "bytes") + .header(header::ETAG, etag) + .body(body) + .unwrap(), + ) + } Err(err) => { tracing::error!("Error creating range stream: {}", err); None // fall through to the full download