diff --git a/.gitignore b/.gitignore index 0d7b9636..905f8a95 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ tests/e2e/playwright/.auth/ # Test fixtures generated on-the-fly by tests/api/run.sh tests/fixtures/chunk-over-cap-*.bin +wasm/oxicloud-hash/target/ diff --git a/Cargo.lock b/Cargo.lock index 2552c1ca..48c87324 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2075,6 +2075,11 @@ name = "fastcdc" version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77af40d8a8dadb92dc178569a5f5edb5f3056e98255c2de48ab5d59a52892e0c" +dependencies = [ + "async-stream", + "tokio", + "tokio-stream", +] [[package]] name = "fastdivide" @@ -4138,7 +4143,6 @@ dependencies = [ "lightningcss", "lru", "md-5 0.11.0", - "memmap2", "mimalloc", "mime_guess", "mockall", diff --git a/Cargo.toml b/Cargo.toml index 3a175cae..d39835e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,8 +68,7 @@ azure_storage = { version = "0.21", default-features = false, features = ["enabl azure_storage_blobs = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } aes-gcm = "0.10.3" lru = "0.16.4" -fastcdc = "4.0.0" -memmap2 = "0.9.10" +fastcdc = { version = "4.0.0", features = ["tokio"] } lettre = { version = "0.11.18", default-features = false, features = ["smtp-transport", "tokio1-rustls-tls", "rustls-native-certs", "builder"] } idna = "1.1" smol_str = { version = "0.3.2", features = ["serde"] } diff --git a/biome.json b/biome.json index 5abc8fe5..9a5abf93 100644 --- a/biome.json +++ b/biome.json @@ -7,7 +7,7 @@ } }, "files": { - "includes": ["static/**/*.js", "static/**/*.css", "static/**/*.json", "!static/js/vendors/"] + "includes": ["static/**/*.js", "static/**/*.css", "static/**/*.json", "!static/js/vendors"] }, "formatter": { "enabled": true, diff --git a/docs/config/env.md b/docs/config/env.md index f131ccd4..87fcfd7f 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -14,8 +14,7 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_MAX_UPLOAD_SIZE` | `10737418240` | Whole-file size ceiling, in bytes (10 GB on 64-bit, 1 GB on 32-bit). Applies to BOTH direct PUTs (per-request body) and chunked uploads (declared `total_size`, checked upfront at session creation). | | `OXICLOUD_DIRECT_PUT_MAX_BYTES` | `1073741824` | Per-request cap for non-chunked PUT bodies, in bytes (1 GiB). Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol (resumable on failure). See [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_CHUNK_MAX_BYTES` | `104857600` | Maximum size of a single chunked-upload PUT in bytes (100 MB). Per-chunk cap, independent of `OXICLOUD_MAX_UPLOAD_SIZE` (whole-file cap). See [Storage Fine Tuning](./storage-fine-tuning.md). | -| `OXICLOUD_UPLOAD_TMPDIR` | (OS temp dir) | Spool directory for non-chunked PUT bodies (`/api/files/upload`, WebDAV PUT). Point at a real-disk path on the same FS as `.blobs/` to avoid tmpfs OOMKill and make blob promotion an atomic rename. See [Storage Fine Tuning](./storage-fine-tuning.md). | -| `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Same-FS / NVMe placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | +| `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Direct (non-chunked) uploads stream straight into the blob store and need no spool directory. Placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_REUSE_PORT` | `false` | Enable `SO_REUSEPORT` so multiple processes can share the same port. **Disabled by default** — a second accidental instance will fail with "address already in use". Enable only for deliberate multi-worker setups (process supervisor, rolling restart). Not supported on Windows. | ## Database diff --git a/docs/config/storage-fine-tuning.md b/docs/config/storage-fine-tuning.md index ea51d3e1..dfd59849 100644 --- a/docs/config/storage-fine-tuning.md +++ b/docs/config/storage-fine-tuning.md @@ -1,62 +1,56 @@ # Storage Fine Tuning -This page is for sysadmins who want to tune **where** OxiCloud spools -upload bodies and **why** the placement matters for throughput and -memory. The defaults work; the gains from a tuned layout are +This page is for sysadmins who want to tune **where** OxiCloud places +upload data on disk and **why** the placement matters for throughput +and memory. The defaults work; the gains from a tuned layout are significant on busy instances or constrained containers. ## The upload lifecycle in 30 seconds -Every upload moves through two stages: - ``` - ┌─── direct (single-PUT) upload ────────┐ -client ─┤ ├──► OxiCloud accepts the - └─── multi-chunk upload │ bytes into a SPOOL on - (`/api/uploads` / │ local disk. - `/dav/uploads/...`) │ - │ Direct upload → OXICLOUD_UPLOAD_TMPDIR - │ Chunked upload → OXICLOUD_CHUNK_DIR - │ - ▼ + ┌─── direct (single-PUT / multipart) ───┐ +client ─┤ │ Streamed DIRECTLY into the + │ ├──► content-addressable store: + │ │ CDC chunking + BLAKE3 + + │ │ dedup checks happen while + │ │ the bytes arrive. No spool + │ │ file, no re-read; chunks + │ │ the store already has are + │ │ never written at all. + │ │ + └─── multi-chunk upload ────────────────┤ Chunk PARTS accumulate on + (`/api/uploads` / │ disk under OXICLOUD_CHUNK_DIR + `/dav/uploads/...`) │ until /complete, which + │ streams them (in order) + ▼ through the same CDC path. ┌─────────────────────────┐ - │ Once the upload is │ - │ complete (and verified │ - │ if a checksum was │ - │ supplied), OxiCloud │ - │ MOVES the assembled │ - │ blob into the configured│ - │ STORAGE BACKEND: │ - │ │ - │ • local FS (.blobs/) │ - │ • S3-compatible │ - │ • Azure Blob │ + │ STORAGE BACKEND │ + │ • local FS (.blobs/) │ + │ • S3-compatible │ + │ • Azure Blob │ └─────────────────────────┘ ``` Two practical consequences: -- **The spool/chunk directories see write-heavy churn** during uploads — - fast disk (NVMe) and sufficient free space matter more here than on - the final storage backend. -- **The promotion from spool → storage is a `rename(2)` whenever - source and destination share a filesystem** (i.e. when the backend - is `local` and the spool dir is on the same FS as `.blobs/`). On - remote backends (S3, Azure) the promotion is always a network - upload from the local spool; placement of the spool still matters - for intake throughput but the "same FS" rule doesn't apply. +- **Direct uploads no longer use a spool directory.** Each uploaded + byte is written at most once — straight into the blob backend as a + CDC chunk. Re-uploads and edited files write only the chunks the + store doesn't already have. +- **The chunk-session directory sees write-heavy churn** during + multi-chunk uploads — fast disk (NVMe) and sufficient free space + matter more here than on the final storage backend. ## Upload size caps — what each one bounds Three independent caps control how large an upload OxiCloud will -accept. Pick them with disk and tmpfs sizing in mind: the spool/chunk -directories must be able to hold the worst case (cap × concurrent -uploads). +accept. Pick them with disk sizing in mind: the chunk directory must +be able to hold the worst case (cap × concurrent sessions). | Variable | Default | What it caps | When it fires | |---|---|---|---| | `OXICLOUD_MAX_UPLOAD_SIZE` | 10 GB | **Whole-file ceiling.** Applies to both direct PUT (per-body) and chunked uploads (declared `total_size`). The absolute upper bound on any single file in OxiCloud. | Chunked: at `POST /api/uploads` against the JSON-declared `total_size`, before any chunk is uploaded. Direct PUT: indirectly via `OXICLOUD_DIRECT_PUT_MAX_BYTES`, which is expected to be ≤ `OXICLOUD_MAX_UPLOAD_SIZE`. | -| `OXICLOUD_DIRECT_PUT_MAX_BYTES` | 1 GiB | **Non-chunked PUT body.** Per-request cap for `POST /api/files/upload`, `PUT /webdav/...`, and `PUT /remote.php/dav/files/.../...`. Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol — which is resumable on failure. | During body streaming, as a per-frame accumulator. Excess → 413 with a "use chunked upload" hint. | +| `OXICLOUD_DIRECT_PUT_MAX_BYTES` | 1 GiB | **Non-chunked PUT body.** Per-request cap for `PUT /webdav/...` and `PUT /remote.php/dav/files/.../...`. Set below `OXICLOUD_MAX_UPLOAD_SIZE` so larger files are pushed onto the chunked protocol — which is resumable on failure. | During body streaming, as a per-frame accumulator. Excess → 413 with a "use chunked upload" hint. | | `OXICLOUD_CHUNK_MAX_BYTES` | 100 MB | **Per-chunk body** in a chunked-upload session (`PATCH /api/uploads/{id}` or `PUT /remote.php/dav/uploads/.../chunk`). Independent of the whole-file cap — a 5 GB file in 100 MB chunks is 50 PATCHes each bounded by this. | During chunk-body streaming. Excess → 413. | ### Recommendation: prefer chunked uploads for large files @@ -64,244 +58,117 @@ uploads). The defaults (`OXICLOUD_DIRECT_PUT_MAX_BYTES` = 1 GiB, well below `OXICLOUD_MAX_UPLOAD_SIZE` = 10 GB) are deliberately asymmetric. Files between those two caps can only succeed via the chunked -protocol. Three reasons to keep them that way: +protocol. The reason is **resilience**: a direct PUT at 95 % of 5 GB +that drops loses everything (the partially ingested chunks are +reclaimed by GC, but the client must restart from byte 0). The same +drop on a chunked upload loses one ~5 MB chunk; the client retries +that chunk and continues. NextCloud desktop and the OxiCloud web UI +already switch to chunked at ~10 MB (`CHUNKED_UPLOAD_THRESHOLD`). -- **Resilience.** A direct PUT at 95 % of 5 GB that drops loses - everything. The same drop on a chunked upload loses one ~5 MB - chunk; the client retries that chunk and continues. -- **Memory + disk pressure.** Direct PUT spools the full body to - disk per request. Ten concurrent 5 GB direct PUTs use up to 50 GB - of transient spool disk. Chunked spreads each upload across many - small PATCHes; per-request resource use stays bounded by - `OXICLOUD_CHUNK_MAX_BYTES`. -- **Convention.** NextCloud desktop and the OxiCloud web UI already - switch to chunked at ~10 MB (`CHUNKED_UPLOAD_THRESHOLD`). +### Disk sizing -### Why caps matter for tmpfs sizing +OxiCloud streams bodies frame-by-frame, so **RAM** is bounded +(~10 MB per in-flight upload for the CDC ingest buffers) regardless +of the caps. **Disk space** scales with the caps: -OxiCloud streams bodies frame-by-frame, so **RAM** is bounded to one -HTTP frame (~64 KB) per request regardless of the caps. **Disk space**, -however, scales with the caps: +- **Direct PUT / multipart**: no transient spool. Bytes land directly + in the blob backend as deduplicated chunks; worst-case extra disk + per upload is the file's own (deduplicated) size — the same space + the stored file occupies afterwards. +- **Chunked upload**: each in-flight session accumulates its chunk + parts under `OXICLOUD_CHUNK_DIR` until `/complete` streams them + into the blob store and the session is cleaned up. Worst case disk + per session = **file_size** (the parts); total = + `OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions`. -- **Direct PUT**: each in-flight upload spools the full body to disk - under `OXICLOUD_UPLOAD_TMPDIR` until promotion. Worst case disk = - `OXICLOUD_DIRECT_PUT_MAX_BYTES × concurrent_direct_PUTs`. -- **Chunked upload**: each in-flight session accumulates chunks - under `OXICLOUD_CHUNK_DIR`, then assembles them into a single temp - file before promotion. Worst case disk per session = **2 × - file_size** (chunks + assembled file); total disk = - `2 × OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions`. +| Settings | Chunked worst case (5 sessions) | Safe on 4 GB volume? | +|---|---|---| +| Defaults: `OXICLOUD_MAX_UPLOAD_SIZE`=10 GB | 50 GB | ❌ overflows | +| `OXICLOUD_MAX_UPLOAD_SIZE`=500 MB | 2.5 GB | ✅ fits | -The chunked formula uses `OXICLOUD_MAX_UPLOAD_SIZE` because that's -what bounds the declared `total_size` at session creation. The -direct-PUT formula uses the smaller `OXICLOUD_DIRECT_PUT_MAX_BYTES` -since that's what bounds each direct PUT body. +### Don't put `OXICLOUD_CHUNK_DIR` on tmpfs -### Sizing examples - -A 4 GB tmpfs serving a small team (5 concurrent direct PUTs OR 5 -concurrent chunked sessions): - -| Settings | Direct-PUT worst case | Chunked worst case | Safe on 4 GB tmpfs? | -|---|---|---|---| -| Defaults: `OXICLOUD_MAX_UPLOAD_SIZE`=10 GB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=1 GiB, `OXICLOUD_CHUNK_MAX_BYTES`=100 MB | 5 GiB (5 × 1 GiB) | 100 GB (5 × 2 × 10 GB) | ❌ chunked overflows | -| `OXICLOUD_MAX_UPLOAD_SIZE`=500 MB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=100 MB, `OXICLOUD_CHUNK_MAX_BYTES`=20 MB | 500 MB | 5 GB | ⚠ direct PUT fits, chunked still overflows | -| `OXICLOUD_MAX_UPLOAD_SIZE`=300 MB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=50 MB, `OXICLOUD_CHUNK_MAX_BYTES`=10 MB | 250 MB | 3 GB | ✅ both fit | - -A real-disk volume (cheap, large): - -| Settings | Direct-PUT worst case | Chunked worst case | Comment | -|---|---|---|---| -| Defaults (see row above) | 5 GiB | 100 GB | Fine on a 200+ GB volume; almost any real-disk setup | -| `OXICLOUD_MAX_UPLOAD_SIZE`=100 GB, `OXICLOUD_DIRECT_PUT_MAX_BYTES`=5 GiB, `OXICLOUD_CHUNK_MAX_BYTES`=500 MB | 25 GiB | 1 TB | Plausible for video archives; needs a dedicated upload volume | - -### Choosing tmpfs vs real disk - -| Constraint | Choice | -|---|---| -| `OXICLOUD_DIRECT_PUT_MAX_BYTES × concurrent_direct_PUTs + 2 × OXICLOUD_MAX_UPLOAD_SIZE × concurrent_chunked_sessions ≤ free RAM × 0.5` | tmpfs OK (fast, atomic with `.blobs/` if also tmpfs) | -| Worst case exceeds half free RAM | **real disk** — same filesystem as `.blobs/` ideal | -| Container with cgroup memory limit | **real disk** — tmpfs spool counts against the cgroup limit and triggers OOMKill | -| Multi-GB uploads expected | **real disk** — even small concurrency on tmpfs runs out of space | -| Small-file workload only (≤ 50 MB), high concurrency | tmpfs gives a noticeable intake speedup | - -The defaults (`OXICLOUD_MAX_UPLOAD_SIZE`=10 GB, -`OXICLOUD_DIRECT_PUT_MAX_BYTES`=1 GiB, -`OXICLOUD_CHUNK_MAX_BYTES`=100 MB) assume **real disk**. Don't run -the defaults against tmpfs unless you've sized it for the worst case. +In many container setups the OS temp dir is **tmpfs** — RAM-backed +storage that counts against the cgroup memory limit. A few concurrent +multi-GB chunked sessions on tmpfs will wake the OOMKiller long +before the uploads finish. Point `OXICLOUD_CHUNK_DIR` at a real-disk +directory in containers. ## TL;DR | Variable | Default | Purpose | |---|---|---| | `OXICLOUD_STORAGE_PATH` | `./storage` | Where `.blobs/` lives (the canonical content store) | -| `OXICLOUD_UPLOAD_TMPDIR` | OS temp dir | Where non-chunked PUT bodies are spooled | | `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Where chunked-upload sessions accumulate | The two rules that matter most: -1. **Put all three on the same filesystem.** Blob promotion is an - atomic `rename(2)` when source and destination share an FS — cheap - and crash-safe. Across filesystems it becomes a full `read + write + - unlink`, multiplying the IO and widening the durability window. -2. **Don't leave the spool dir on tmpfs** (the default in many - containers). Spool bodies count against the cgroup memory limit +1. **Keep `OXICLOUD_CHUNK_DIR` off tmpfs** (the default in many + containers) — chunk parts count against the cgroup memory limit and can trigger OOMKill on multi-GB uploads. +2. **NVMe for the chunk dir pays off** on deployments with heavy + large-file traffic: each chunk PUT writes a file and the progress + bitmap, and `/complete` reads them all back in order. -## Where each upload surface spools - -OxiCloud has several entry points that accept request bodies. They -land in different places by default: +## Where each upload surface writes | Surface | Default destination | Configurable via | |---|---|---| | REST chunked PUT (`PATCH /api/uploads/{id}`) | `{STORAGE_PATH}/.uploads/{upload_id}/chunk_NNNNNN` | `OXICLOUD_CHUNK_DIR` | -| REST chunked assemble (during `/complete`) | `{STORAGE_PATH}/.uploads/{upload_id}/assembled` | `OXICLOUD_CHUNK_DIR` | | NextCloud chunked PUT (`PUT /dav/uploads/.../chunk`) | `{STORAGE_PATH}/.uploads/nextcloud/{user}/{upload_id}/{chunk_name}` | `OXICLOUD_CHUNK_DIR` | -| NextCloud chunked assemble (during `MOVE`) | `{STORAGE_PATH}/.uploads/nextcloud/{user}/{upload_id}/.assembled` | `OXICLOUD_CHUNK_DIR` | -| Native WebDAV PUT (`PUT /webdav/{path}`) | OS temp dir (`/tmp`) | `OXICLOUD_UPLOAD_TMPDIR` | -| NextCloud single-file PUT (`PUT /dav/files/.../{path}`) | OS temp dir | `OXICLOUD_UPLOAD_TMPDIR` | -| REST multipart upload (`POST /api/files/upload`) | `{STORAGE_PATH}/.dedup_temp/upload-{uuid}` | `OXICLOUD_STORAGE_PATH` (subdir is hard-wired) | -| Final blob storage (after fsync + rename) | `{STORAGE_PATH}/.blobs/{ab}/{abc…}.blob` | `OXICLOUD_STORAGE_PATH` | +| Direct PUT / multipart / WOPI / chunked `/complete` | straight into the blob backend (CDC chunks) | `OXICLOUD_STORAGE_PATH` (local backend) | +| Final blob storage | `{STORAGE_PATH}/.blobs/{ab}/{abc…}.blob` | `OXICLOUD_STORAGE_PATH` | -## Why placement matters - -### 1. Same filesystem ⇒ promotion is a rename - -OxiCloud uses **content-addressable storage**: the final blob path is -derived from the file's BLAKE3 hash, which can only be known after the -last byte arrives. So every upload writes to a temp location first, -then **promotes** the temp file to `.blobs/{ab}/{abc…}.blob` by way of -a `rename(2)` call. - -- **Same FS:** `rename` is atomic, O(1), no data copy. Total upload - cost = body bytes received + one rename syscall. Crash-safe — the - blob either exists at the final path or doesn't. -- **Cross-FS:** the kernel can't `rename(2)` across filesystems. The - blob backend falls back to `fs::copy + fs::remove_file` (visible in - `local_blob_backend.rs` as the EXDEV handler). Total cost = body - bytes received + one full file copy. Doubles the IO bandwidth used - per upload and widens the durability window. - -### 2. Spool off tmpfs - -`tempfile::NamedTempFile::new()` (used when `OXICLOUD_UPLOAD_TMPDIR` -is unset) honors `$TMPDIR`, which in many container setups points at -**tmpfs** — RAM-backed storage. A 2 GB upload spool then consumes 2 GB -of memory until the rename promotes it to disk. - -In a Kubernetes pod with a 4 GB memory limit, the OOMKiller wakes up -long before the upload finishes. With `OXICLOUD_UPLOAD_TMPDIR` pointed -at a real-disk directory, the spool's memory footprint stays at ~one -HTTP frame regardless of file size. - -### 3. NVMe for the hot path - -The chunked-upload session directory sees a LOT of small writes — -each chunk PUT writes a file, the progress bitmap is rewritten after -each PUT, the assemble step reads them all back in order. Pointing -`OXICLOUD_CHUNK_DIR` at an NVMe device is a substantial win on -deployments that handle large file uploads, even if the final blob -storage is on slower disk. - -The same applies to `OXICLOUD_UPLOAD_TMPDIR` (single-file PUTs). - -A common high-throughput layout: - -- **NVMe** (small, fast): `OXICLOUD_CHUNK_DIR`, `OXICLOUD_UPLOAD_TMPDIR` -- **HDD or NAS** (large, cheap): `OXICLOUD_STORAGE_PATH`/`.blobs/` - -Trade-off: the rename optimization (rule 1) DOESN'T apply across -filesystems. If you split the hot path off the blob filesystem, every -upload pays a full file copy on promotion. You have to choose -between **fast intake** and **zero-copy promotion**. - -| Goal | Layout | Cost per upload | -|---|---|---| -| Fastest possible intake | NVMe chunk dir + HDD blobs | 1× write to NVMe + 1× read NVMe + 1× write to HDD (copy) | -| Lowest IO + crash safety | NVMe everything OR HDD everything | 1× write to disk + 1 rename (~0 cost) | -| Default (do nothing) | Everything under `STORAGE_PATH` on whatever FS that is | Depends on `STORAGE_PATH` placement | - -For most deployments **"same FS everywhere"** wins. The NVMe-split is -useful when intake latency dominates the user experience and you can -afford the doubled IO. +The local blob backend stages each chunk write under +`{STORAGE_PATH}/.dedup_temp/` and promotes it with an atomic +`rename(2)` — both directories live under `OXICLOUD_STORAGE_PATH`, +so same-filesystem placement (and therefore atomic promotion) is +automatic and not separately configurable. ## Recommended layouts ### Single-disk box (most common) -Defaults are fine. Optionally set `OXICLOUD_UPLOAD_TMPDIR` to keep -the PUT spool off `/tmp`: +Defaults are fine: ```bash OXICLOUD_STORAGE_PATH=/var/lib/oxicloud -OXICLOUD_UPLOAD_TMPDIR=/var/lib/oxicloud/.spool # OXICLOUD_CHUNK_DIR unset → /var/lib/oxicloud/.uploads ``` -All three on the same filesystem → rename promotion → atomic and fast. - ### Container with constrained memory -Critical: make sure neither spool sits on tmpfs. +Critical: make sure the chunk dir doesn't sit on tmpfs. ```bash OXICLOUD_STORAGE_PATH=/data -OXICLOUD_UPLOAD_TMPDIR=/data/.spool OXICLOUD_CHUNK_DIR=/data/.uploads ``` -If you can't mount a writable `/data`, at minimum bind-mount a real -volume at the spool dirs. - ### Split-disk (NVMe intake + HDD blobs) ```bash OXICLOUD_STORAGE_PATH=/mnt/hdd/oxicloud # .blobs/ + .dedup_temp/ -OXICLOUD_UPLOAD_TMPDIR=/mnt/nvme/oxi-spool OXICLOUD_CHUNK_DIR=/mnt/nvme/oxi-chunks ``` -Faster intake; pays a copy on promotion. Worth it when uploads are -many small files (NVMe IOPS dominates) or when intake latency directly -hits user-visible UX. +Chunk parts land on NVMe (fast PUTs, fast `/complete` read-back); +the deduplicated chunks are written once to the HDD-backed blob +store as `/complete` streams through them. -## Sharing the spool and chunk directories +## Sharing the chunk directory -Pointing `OXICLOUD_UPLOAD_TMPDIR` and `OXICLOUD_CHUNK_DIR` at the -**same directory** is supported by design. Each writer tags its -output so the surfaces never interfere with each other: +The REST and NC chunked surfaces can share `OXICLOUD_CHUNK_DIR` by +design. Each writer tags its output so they never interfere: | Writer | On-disk name pattern | |---|---| -| PUT spool (single-file uploads) | `.tmpXXXXXXXX` — files (not directories), random suffix | | REST chunked sessions | `oxi-chunk-{uuid}/` — directories with a well-known prefix | | NC chunked subtree | `nextcloud/{user}/{uuid}/` — under its own root subdir | The 24-hour orphan-session cleanup loop filters strictly on the `oxi-chunk-` prefix, so it can NEVER delete a non-OxiCloud directory -that happens to live alongside chunked sessions. The PUT spool's -`.tmpXXXX` files are files (not directories) and the NC subtree's -`nextcloud/` root has its own name — both are invisible to the -cleanup loop. - -**Recommendation:** for new deployments, use separate directories -anyway (the defaults `.spool/` and `.uploads/` already do this) — -it makes disk-usage attribution clearer and keeps IOPS isolated when -both are busy. Shared directories are safe to use when disk layout -forces it. - -## What's NOT yet configurable - -- **REST multipart upload directory** (`POST /api/files/upload`) is - hard-wired to `{STORAGE_PATH}/.dedup_temp/`. It can't be moved - separately. Same-FS placement is automatic. -- **WOPI PutFile spool** (Office editor saves) uses the bare OS temp - dir without honoring `OXICLOUD_UPLOAD_TMPDIR`. This is a known - inconsistency and on the hardening backlog. -- **Per-user / per-drive spool directories** — all users share the - same `OXICLOUD_CHUNK_DIR` root today. Multi-tenant isolation - through separate spool dirs isn't supported. +that happens to live alongside chunked sessions. ## Quick verification diff --git a/docs/delta-upload-protocol.md b/docs/delta-upload-protocol.md new file mode 100644 index 00000000..686198d5 --- /dev/null +++ b/docs/delta-upload-protocol.md @@ -0,0 +1,172 @@ +# Delta-Upload Protocol + +Upload only what changed. The server's dedup store already splits every +file into content-defined chunks (FastCDC, 64 KB – 1 MiB, avg 256 KB, +BLAKE3-addressed) and shares unchanged chunks between file versions — +but a classic upload still transfers every byte just for the server to +discard the known ones. This protocol moves the "which chunks are new?" +question to the client, so unchanged bytes never cross the wire. + +Editing a few bytes of a 500 MB file re-uploads ~1 MiB instead of +500 MB. + +## Who can use it + +Any authenticated API client. The OxiCloud web frontend uses it +automatically for files ≥ 8 MiB (`features/files/deltaUpload.js` + +`workers/deltaWorker.js`, chunking with the vendored WASM build of the +server's own FastCDC+BLAKE3 crates, falling back to a plain byte upload +on any failure). Generic WebDAV/NextCloud clients cannot (their +protocols have no delta concept) — they keep uploading full bytes, and +the server keeps deduplicating those on write. + +Chunk boundaries are the **client's choice**: matching the server's +FastCDC parameters (64 KB / 256 KB / 1 MiB, as the bundled WASM module +does) maximizes cross-version sharing — including against versions that +entered through plain byte uploads — but any split with chunks of +1 byte … 1 MiB is valid; correctness is guaranteed by server-side +verification, not by the chunking scheme. + +## The three steps + +### 1. `POST /api/files/delta/negotiate` + +```json +{ "chunks": [ { "h": "", "s": 262144 }, … ] } +``` + +Response — the distinct chunk hashes the caller must upload, in +first-occurrence order: + +```json +{ "missing": [ "", … ] } +``` + +The answer is **user-scoped**: a chunk counts as available only when one +of the *caller's own* (non-trashed) files already references it. The +endpoint is purely advisory — the commit re-checks entitlement +atomically, so a stale or spoofed answer can never leak content. + +### 2. `PUT /api/files/delta/chunks` + +Body: `application/octet-stream`, a sequence of frames + +``` +[u32 length, big-endian][length bytes] …repeated… +``` + +- one frame per chunk, each 1 byte … 1 MiB (the CDC maximum), +- whole request capped by `OXICLOUD_CHUNK_MAX_BYTES` (default 100 MB) — + split larger deltas across requests. + +The server **recomputes BLAKE3 of every frame itself** (a declared hash +is never trusted for content addressing) and registers the chunks as +unreferenced orphans (`ref_count = 0`). Response: + +```json +{ "received": [ { "h": "", "s": 262144 }, … ] } +``` + +Compare against your own hashes to catch corruption before committing. +Abandoned uploads need no cleanup call: the periodic GC sweeps +zero-reference chunks. + +### 3. `POST /api/files/delta/commit` + +```json +{ + "file_hash": "", + "chunks": [ { "h": "…", "s": 262144 }, … ], // full sequence, in order + "name": "video.mp4", "folder_id": "" // create mode + // — or — + "file_id": "" // update (replace content) +} +``` + +Server-side, in order: + +1. **AuthZ** — `Create` on the folder (create mode) or `Update` on the + file (update mode); quota on the logical size. +2. **Pin** — one atomic `UPDATE … RETURNING` takes a reference on every + distinct chunk the caller is *entitled* to: chunks reachable through + the caller's own files, or unreferenced orphans (the just-uploaded + state). Anything else → `409 { "still_missing": […] }`: upload + exactly those and retry the same commit. +3. **Verify** — the pinned sequence is re-read and the whole-file BLAKE3 + recomputed. A mismatch releases the pins and returns 400 (and an + audit event): the declared `file_hash` is never trusted, because a + forged manifest would poison future whole-file dedup hits for *other + users* uploading the genuine content. +4. **Attach** — the manifest is inserted with the same accounting as the + streaming byte path (a concurrent identical commit resolves via + `ON CONFLICT`: the loser's references are released and it becomes a + dedup hit). +5. **Row** — the file is created (`201`, body = FileDto) or its content + swapped (`200`). + +If the caller already owns the exact `file_hash`, the commit +short-circuits to a pure reference bump — chunks aren't even looked at +(same as `POST /api/files/by-hash`). + +## Delta download (sync clients) + +The inverse direction, for a client app that already holds an older +version locally and wants the server's current one: + +1. `GET /api/files/{id}/manifest` → `{ file_hash, total_size, chunks }` + — the file's chunk recipe. **Owner-scoped** like the rest of the + delta surface (shared files use the regular download endpoints). + Served with `ETag: ""`; a manifest is immutable for a + given hash, so `If-None-Match` revalidation answers 304 — polling + sync clients pay one header round-trip per unchanged file. +2. Diff the manifest against the local chunk inventory (chunk the local + copy with the same WASM module the upload direction ships). +3. `POST /api/files/delta/download` with `{ "hashes": […] }` → the + requested chunks as `[u32 BE length][bytes]` frames in request order + (the same wire format as the upload direction). Entitlement is the + same possession rule as negotiate/commit: chunks must be reachable + through the caller's own files; anything else → 404 + `{ "not_available": […] }` — deliberately indistinguishable from + "never existed". Batches are bounded by `OXICLOUD_CHUNK_MAX_BYTES`; + split large deltas across requests. +4. Reassemble locally per the manifest order and verify the whole-file + BLAKE3 against `file_hash`. + +Editing 3 bytes of a 24 MB file on one device costs a second device one +manifest GET plus ~1 chunk (~256 KB) instead of 24 MB. + +## Security model + +- **No content oracle.** Possession is proven per chunk: without bytes + you can only claim what your own files already reference. Probing + someone else's chunk hashes yields `still_missing`, indistinguishable + from the hash never existing. +- **No manifest poisoning.** `file_hash` and every chunk hash are + recomputed server-side before becoming addressable. +- **Bounded resources.** Per-frame cap 1 MiB, per-request cap + `OXICLOUD_CHUNK_MAX_BYTES`, whole-file cap `OXICLOUD_MAX_UPLOAD_SIZE`, + per-caller rate limit (240 delta requests/min), quota enforced at + commit. Orphan chunks are GC-swept. +- **Audit.** Rejections emit `delta_upload.rejected` with stable + `reason` keys: `rate_limited`, `chunk_verification_failed`, + `file_hash_mismatch` — and `delta_download.rejected` with + `manifest_not_owner` / `chunks_not_owned`. AuthZ denials surface as + the engine's standard `authz.denied`. + +## Error summary + +| Status | Meaning | Client action | +|---|---|---| +| 400 | malformed framing/hashes/sizes, or `file_hash` mismatch | fix and retry from step 1 | +| 404 | folder/file not found or not accessible | — | +| 409 | `{"still_missing": […]}` | PUT those chunks, retry the commit | +| 429 | rate limited | back off | +| 507 | quota exceeded | — | + +## Cost notes + +- `negotiate` is one indexed query (GIN over manifest chunk arrays). +- `commit` performs one sequential server-side read of the full logical + file for verification — cheap on local backends, a full object read on + S3/Azure. Still strictly cheaper than receiving the bytes, and the + client's bandwidth saving is unaffected. diff --git a/example.env b/example.env index 126803e0..ea78f21a 100644 --- a/example.env +++ b/example.env @@ -55,22 +55,14 @@ OXICLOUD_SERVER_HOST=127.0.0.1 #OXICLOUD_CHUNK_MAX_BYTES=104857600 # ── Upload spool directories ───────────────────────────────────────── -# Where in-flight uploads land BEFORE being promoted into final blob -# storage. Same-filesystem placement (with `.blobs/`) makes the -# promotion an atomic rename(2); NVMe placement speeds up intake. +# Direct (non-chunked) uploads stream straight into the blob store — +# no spool directory. Only chunked-upload sessions accumulate on disk. # See docs/config/storage-fine-tuning.md for layout examples. -# Directory for non-chunked PUT spool tempfiles. Default: OS temp dir -# ($TMPDIR / /tmp), often tmpfs (RAM) in containers — writing a large -# upload there fills page-cache that counts against the cgroup memory -# limit and can OOMKill the process. Point this at a real-disk path -# (same FS as the storage backend is ideal). -#OXICLOUD_UPLOAD_TMPDIR=/var/lib/oxicloud/tmp - # Root directory for chunked-upload sessions (REST + NextCloud chunked -# share this root). Default: {STORAGE_PATH}/.uploads. Pointing this at -# NVMe accelerates the chunk-write + assembly loop; pointing it at the -# same FS as `.blobs/` makes blob promotion atomic. +# share this root). Default: {STORAGE_PATH}/.uploads. Avoid tmpfs in +# containers (parts count against the cgroup memory limit); NVMe +# placement accelerates chunk PUTs and the /complete streaming pass. #OXICLOUD_CHUNK_DIR=/var/lib/oxicloud/.uploads # How often (seconds) the background sweep reconciles each user's cached diff --git a/migrations/20260628000000_delta_upload_gin_index.sql b/migrations/20260628000000_delta_upload_gin_index.sql new file mode 100644 index 00000000..ef742b16 --- /dev/null +++ b/migrations/20260628000000_delta_upload_gin_index.sql @@ -0,0 +1,13 @@ +-- Delta-upload protocol: chunk-level ownership lookups. +-- +-- The negotiate/commit endpoints answer "which of these N chunk hashes may +-- this caller claim without uploading bytes?" — a chunk is claimable when a +-- manifest of one of the caller's (non-trashed) files contains it. That +-- containment test (`chunk_hashes @> ARRAY[hash]`) would be a sequential +-- scan over storage.chunk_manifests without an index; GIN makes each probe +-- an index lookup. +-- +-- Plan B if this disappoints at scale: a normalized +-- storage.manifest_chunks(file_hash, chunk_hash) join table. +CREATE INDEX IF NOT EXISTS idx_chunk_manifests_chunk_hashes_gin + ON storage.chunk_manifests USING GIN (chunk_hashes); diff --git a/scripts/build-wasm.sh b/scripts/build-wasm.sh new file mode 100755 index 00000000..9a64cff7 --- /dev/null +++ b/scripts/build-wasm.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Rebuild the vendored BLAKE3 WASM module (static/js/vendors/hash-wasm/). +# +# The generated artifacts ARE committed — like the other vendored modules +# (pdf.js) — so regular frontend/backend builds never need the wasm +# toolchain. Re-run this script only when wasm/oxicloud-hash/ changes +# (e.g. bumping the blake3 crate, or adding FastCDC for the delta-sync +# client) and commit the regenerated files. +# +# Requirements (one-time): +# rustup target add wasm32-unknown-unknown +# cargo install wasm-bindgen-cli --locked +# +# wasm-bindgen-cli's version must match the crate's `wasm-bindgen` +# dependency; cargo prints a clear error when they drift. + +set -euo pipefail +cd "$(dirname "$0")/.." + +CRATE=wasm/oxicloud-hash +OUT=static/js/vendors/hash-wasm + +# SIMD128 is baseline in every evergreen browser (Chrome 91+, Firefox 89+, +# Safari 16.4+) and is worth ~3-4× in hashing throughput. Browsers without +# it fail instantiation; the frontend detects that and falls back to a +# plain byte upload. +RUSTFLAGS="-C target-feature=+simd128" \ + cargo build \ + --manifest-path "$CRATE/Cargo.toml" \ + --target wasm32-unknown-unknown \ + --release + +wasm-bindgen \ + --target web \ + --no-typescript \ + --out-dir "$OUT" \ + "$CRATE/target/wasm32-unknown-unknown/release/oxicloud_hash_wasm.wasm" + +echo "Vendored: $(ls -la "$OUT" | tail -n +2 | awk '{print $9, "("$5" bytes)"}' | xargs)" diff --git a/src/application/ports/chunked_upload_ports.rs b/src/application/ports/chunked_upload_ports.rs index 5d7d5427..098dd276 100644 --- a/src/application/ports/chunked_upload_ports.rs +++ b/src/application/ports/chunked_upload_ports.rs @@ -98,6 +98,21 @@ pub struct UploadStatusResponseDto { pub is_complete: bool, } +/// A completed upload session, ready to be streamed into the blob store. +/// +/// `chunk_paths` lists every chunk file in assembly order — the caller +/// concatenates them as one byte stream (CDC chunking + hashing happen in +/// that single pass). The files stay on disk until `finalize_upload`, so a +/// failed completion (e.g. checksum mismatch) remains retryable. +#[derive(Debug, Clone)] +pub struct CompletedUploadParts { + pub chunk_paths: Vec, + pub filename: String, + pub folder_id: Option, + pub content_type: String, + pub total_size: u64, +} + /// Port for chunked/resumable upload operations. /// /// Implementations manage upload sessions, chunk storage, reassembly, @@ -137,16 +152,15 @@ pub trait ChunkedUploadPort: Send + Sync + 'static { user_id: Uuid, ) -> Result; - /// Assemble all chunks into the final file. + /// Validate completion and hand back the ordered chunk parts. /// - /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`. - /// The hash is computed during assembly (hash-on-write), eliminating a - /// second sequential read of the assembled file. + /// No assembled file is written — the caller streams the parts straight + /// into the content-addressable store. async fn complete_upload( &self, upload_id: &str, user_id: Uuid, - ) -> Result<(PathBuf, String, Option, String, u64, String), DomainError>; + ) -> Result; /// Finalize upload: clean up the session and temporary files. async fn finalize_upload(&self, upload_id: &str, user_id: Uuid) -> Result<(), DomainError>; diff --git a/src/application/ports/dedup_ports.rs b/src/application/ports/dedup_ports.rs index 95f219ea..23ddc962 100644 --- a/src/application/ports/dedup_ports.rs +++ b/src/application/ports/dedup_ports.rs @@ -80,18 +80,6 @@ pub struct DedupStatsDto { /// duplicate storage automatically. Multiple file references can /// point to the same physical blob. pub trait DedupPort: Send + Sync + 'static { - /// Store content with deduplication (streaming from file). - /// - /// If `pre_computed_hash` is provided (e.g. hash-on-write from the handler), - /// the file will NOT be re-read to calculate the hash — saving one full - /// sequential read of the file. - async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result; - /// Check if a blob with the given hash exists. async fn blob_exists(&self, hash: &str) -> bool; diff --git a/src/application/ports/file_ports.rs b/src/application/ports/file_ports.rs index b40e0a35..19d9e926 100644 --- a/src/application/ports/file_ports.rs +++ b/src/application/ports/file_ports.rs @@ -1,6 +1,5 @@ use bytes::Bytes; use futures::Stream; -use std::path::Path; use std::pin::Pin; use std::sync::Arc; use uuid::Uuid; @@ -17,74 +16,51 @@ use crate::domain::services::authorization::Permission; // Upload port // ───────────────────────────────────────────────────── +/// A blob already stored in the content-addressable chunk store, carrying +/// exactly ONE reference that the receiving method takes ownership of. +/// +/// Produced by the upload-ingest layer (`interfaces::upload_ingest`), which +/// streams the request body straight into the CDC dedup store. Methods that +/// accept a `StoredBlob` either attach the reference to a file row or +/// release it on failure — callers never need to compensate themselves. +#[derive(Debug, Clone)] +pub struct StoredBlob { + /// BLAKE3 of the full content (manifest / blob key). + pub hash: String, + /// Content size in bytes. + pub size: u64, + /// `false` when the content already existed (dedup hit) — forwarded to + /// lifecycle hooks so e.g. thumbnails aren't regenerated for known blobs. + pub is_new_blob: bool, +} + /// Primary port for file upload operations. /// -/// **All upload paths converge on streaming-to-disk** — no method accepts -/// `Vec` for content. Even `create_file` / `update_file` (WebDAV -/// helpers that receive `&[u8]`) spool to a temp file internally so that -/// peak RAM stays at ~256 KB regardless of file size. -/// -/// - Normal uploads: handler spools multipart to temp file → `upload_file_streaming` -/// - Chunked uploads: chunks already on disk → `upload_file_from_path` -/// - WebDAV PUT (new): handler streams to temp file → `update_file_streaming` -/// - WebDAV PUT (small/compat): `create_file` / `update_file` spool internally +/// **All upload paths converge on streaming-into-the-chunk-store** — content +/// never passes through this port; it is ingested by the interface layer +/// (CDC chunking + hashing while the body arrives, no spool file) and only +/// the resulting [`StoredBlob`] reference travels through here. pub trait FileUploadUseCase: Send + Sync + 'static { - /// Upload from a temp file already on disk (true streaming, ~256 KB RAM). + /// Register a new file row pointing at an already-ingested blob. /// - /// When `pre_computed_hash` is `Some`, the blob store skips the hash - /// re-read — the handler already computed it during the multipart spool. + /// Takes ownership of the blob's reference (released on failure). async fn upload_file_streaming( &self, name: String, folder_id: Option, content_type: String, - temp_path: &Path, - size: u64, - pre_computed_hash: Option, + blob: StoredBlob, ) -> Result; - /// Upload from a file already assembled on disk (chunked uploads). + /// Replace the content of the file at `path` with an already-ingested + /// blob, or create the file when it doesn't exist (WebDAV/WOPI PUT). /// - /// Same as `upload_file_streaming` but with a separate name for clarity. - async fn upload_file_from_path( - &self, - name: String, - folder_id: Option, - content_type: String, - file_path: &Path, - pre_computed_hash: Option, - ) -> Result; - - /// Creates a new file at the specified path (for WebDAV) - async fn create_file( - &self, - parent_path: &str, - filename: &str, - content: &[u8], - content_type: &str, - ) -> Result; - - /// Updates the content of an existing file (for WebDAV) - async fn update_file( - &self, - path: &str, - content: &[u8], - content_type: &str, - modified_at: Option, - ) -> Result; - - /// Streaming update — spools body to a temp file with incremental hash, - /// then atomically replaces the file content via dedup store. - /// - /// Peak RAM: ~256 KB regardless of file size. - /// Used by WebDAV PUT for large files. + /// Takes ownership of the blob's reference (released on failure). async fn update_file_streaming( &self, path: &str, - temp_path: &Path, - size: u64, + blob: StoredBlob, content_type: &str, - pre_computed_hash: Option, modified_at: Option, ) -> Result; } diff --git a/src/application/ports/storage_ports.rs b/src/application/ports/storage_ports.rs index 213cb759..33307c35 100644 --- a/src/application/ports/storage_ports.rs +++ b/src/application/ports/storage_ports.rs @@ -251,21 +251,21 @@ pub struct CopyFolderTreeResult { /// Secondary port for file **writing**. /// -/// Covers: upload (buffered + streaming), move, delete, update, -/// and deferred registration for the write-behind cache. +/// Covers: upload registration, move, delete, update, and deferred +/// registration for the write-behind cache. pub trait FileWritePort: Send + Sync + 'static { - /// Streaming upload — saves a file from a temp file already on disk. + /// Register a file row pointing at a blob already stored in the + /// content-addressable chunk store. /// - /// When `pre_computed_hash` is provided, the dedup service skips the - /// hash re-read — zero extra I/O beyond the initial spool. - async fn save_file_from_temp( + /// Takes ownership of one blob reference: on any failure the reference + /// is released before the error is returned. + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result; /// Moves a file to another folder. @@ -281,22 +281,20 @@ pub trait FileWritePort: Send + Sync + 'static { /// Deletes a file. async fn delete_file(&self, id: &str) -> Result<(), DomainError>; - /// Streaming update — replaces file content from a temp file on disk. + /// Atomically swap a file's content to a blob already stored in the + /// content-addressable chunk store. /// - /// When `pre_computed_hash` is provided, the dedup service skips the - /// hash re-read — zero extra I/O beyond the initial spool. - /// Peak RAM: ~256 KB regardless of file size. + /// Takes ownership of one blob reference (released on failure); the + /// previous content's reference is dropped after the swap. /// /// Returns `(new_blob_hash, updated_at_epoch)` — everything a caller /// needs to rebuild the fresh entity/ETag from a `File` it already /// holds, without re-reading the row it just updated. - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, file_id: &str, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - content_type: Option, - pre_computed_hash: Option, modified_at: Option, ) -> Result<(String, i64), DomainError>; diff --git a/src/application/services/delta_upload_service.rs b/src/application/services/delta_upload_service.rs new file mode 100644 index 00000000..07e4312a --- /dev/null +++ b/src/application/services/delta_upload_service.rs @@ -0,0 +1,662 @@ +//! Delta-upload protocol — "upload only what changed". +//! +//! The CDC dedup store already shares unchanged chunks between file +//! versions *after* the bytes arrive; this protocol moves that detection +//! to the client side so unchanged bytes never cross the wire: +//! +//! 1. `negotiate`: the client sends the chunk hashes that compose its +//! file; the server answers which of them it cannot claim — only those +//! need uploading. +//! 2. `chunks`: the client uploads the missing chunks (raw frames). The +//! server recomputes every hash itself and registers the chunks as +//! unreferenced (`ref_count = 0`) orphans — pinned by the commit that +//! follows, or swept by the periodic GC if the client never returns. +//! 3. `commit`: the server pins one reference per distinct chunk (only +//! chunks the caller owns or unreferenced orphans — see the security +//! notes in `dedup_service.rs`), **re-reads the proposed sequence and +//! recomputes the whole-file BLAKE3** (a declared hash is never +//! trusted: a forged manifest would poison future whole-file dedup +//! hits for other users), attaches the manifest with the same +//! accounting as the streaming ingest, and creates or updates the +//! file row. +//! +//! Stateless by design: there is no session table. Every step re-derives +//! its facts from the chunk store, and the GC reclaims anything a client +//! abandons mid-protocol. + +use std::collections::HashSet; +use std::sync::Arc; + +use bytes::Bytes; +use futures::Stream; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; +use uuid::Uuid; + +use crate::application::dtos::file_dto::FileDto; +use crate::application::ports::authorization_ports::AuthorizationEngine; +use crate::application::ports::file_ports::{FileUploadUseCase, StoredBlob}; +use crate::application::ports::storage_ports::{FileReadPort, StorageUsagePort}; +use crate::application::services::file_upload_service::FileUploadService; +use crate::application::services::storage_usage_service::StorageUsageService; +use crate::common::errors::DomainError; +use crate::common::mime_detect::{MAGIC_BYTES_LEN, refine_content_type}; +use crate::domain::services::authorization::{Permission, Resource, Subject}; +use crate::infrastructure::repositories::pg::FileBlobReadRepository; +use crate::infrastructure::services::dedup_service::{CDC_MAX_CHUNK, DedupService}; +use crate::infrastructure::services::pg_acl_engine::PgAclEngine; + +// ── Wire DTOs ──────────────────────────────────────────────────────────────── + +/// One chunk reference: `h` = BLAKE3 hex (64 chars), `s` = size in bytes. +/// Field names are deliberately terse — a 10 GB file is ~40 000 of these. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ChunkRef { + /// BLAKE3 hash of the chunk (64 hex chars). + pub h: String, + /// Chunk size in bytes (1 ..= 1 MiB). + pub s: u64, +} + +/// Request body of `POST /api/files/delta/negotiate`. +#[derive(Debug, Deserialize, ToSchema)] +pub struct DeltaNegotiateRequest { + /// The file's chunks, in order (duplicates allowed — repeated content). + pub chunks: Vec, +} + +/// Response of `POST /api/files/delta/negotiate`. +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaNegotiateResponse { + /// Distinct chunk hashes the caller must upload (first-occurrence order). + pub missing: Vec, +} + +/// Response of `PUT /api/files/delta/chunks` — the server-computed identity +/// of every received frame, in wire order. Clients compare against their +/// own hashes to detect corruption before committing. +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaChunksResponse { + pub received: Vec, +} + +/// Request body of `POST /api/files/delta/commit`. +/// +/// Exactly one of (`name` + `folder_id`) or `file_id` selects the mode: +/// create a new file, or replace an existing file's content. +#[derive(Debug, Deserialize, ToSchema)] +pub struct DeltaCommitRequest { + /// BLAKE3 of the complete file (verified server-side, never trusted). + pub file_hash: String, + /// Full chunk sequence, in file order (per occurrence). + pub chunks: Vec, + /// Create mode: file name (basename). + pub name: Option, + /// Create mode: target folder (caller needs Create permission). + pub folder_id: Option, + /// Update mode: file whose content is replaced (caller needs Write). + pub file_id: Option, +} + +/// Response of `GET /api/files/{id}/manifest` — the recipe to rebuild the +/// file from chunks. Immutable for a given `file_hash`, so clients cache +/// it keyed by hash (the endpoint also serves it with `ETag: file_hash`). +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaManifestResponse { + /// BLAKE3 of the whole file (verify the local reassembly against it). + pub file_hash: String, + /// Total size in bytes. + pub total_size: u64, + /// Full chunk sequence, in file order (per occurrence). + pub chunks: Vec, +} + +/// Request body of `POST /api/files/delta/download` — distinct chunk +/// hashes to fetch, served back as `[u32 BE length][bytes]` frames in +/// request order (the same wire format the upload direction uses). +#[derive(Debug, Deserialize, ToSchema)] +pub struct DeltaDownloadRequest { + pub hashes: Vec, +} + +/// Outcome of a chunk-download authorization. +pub enum DeltaDownloadOutcome { + /// Every requested chunk is servable: `(hash, size)` in request order. + Ready(Vec<(String, u64)>), + /// Some chunks are not available to this caller (not reachable through + /// their files, or unknown — deliberately indistinguishable). + NotAvailable(Vec), +} + +/// Resolved commit mode after request validation. +enum CommitMode { + Create { name: String, folder_id: String }, + Update { file_id: String }, +} + +/// Outcome of a commit attempt. +pub enum DeltaCommitOutcome { + /// The file row exists; `created` distinguishes 201 from 200. + Done { file: FileDto, created: bool }, + /// Some chunks could not be pinned (GC race, skipped negotiate, or + /// chunks the caller may not claim). The client uploads exactly these + /// and retries the same commit. + StillMissing(Vec), +} + +// ── Service ────────────────────────────────────────────────────────────────── + +/// Orchestrates the three delta-upload steps. All authorization lives here +/// (service layer), per the project's AuthZ rule; handlers only +/// authenticate, rate-limit and translate the wire format. +pub struct DeltaUploadService { + dedup: Arc, + uploads: Arc, + file_read: Arc, + quota: Arc, + authz: Arc, + /// Whole-file ceiling — same `max_upload_size` that bounds byte uploads. + max_total_size: u64, + /// Per-request ceiling for batched chunk downloads — same budget as + /// the chunk-upload requests (`chunk_max_bytes`). + max_download_batch: u64, +} + +impl DeltaUploadService { + #[allow(clippy::too_many_arguments)] + pub fn new( + dedup: Arc, + uploads: Arc, + file_read: Arc, + quota: Arc, + authz: Arc, + max_total_size: u64, + max_download_batch: u64, + ) -> Self { + Self { + dedup, + uploads, + file_read, + quota, + authz, + max_total_size, + max_download_batch, + } + } + + /// Most chunks a single request may reference: the whole-file ceiling + /// divided by the smallest possible CDC chunk, with headroom for + /// fixed-size client chunkers. + fn max_chunk_count(&self) -> usize { + (self.max_total_size as usize + / crate::infrastructure::services::dedup_service::CDC_MIN_CHUNK) + .saturating_mul(2) + .max(1024) + } + + /// Shape-validate a chunk list: hash format, per-chunk size bounds, + /// count and total ceilings. Returns the total size. + fn validate_chunk_list(&self, chunks: &[ChunkRef]) -> Result { + if chunks.len() > self.max_chunk_count() { + return Err(DomainError::validation_error(format!( + "Too many chunks: {} (maximum {})", + chunks.len(), + self.max_chunk_count() + ))); + } + let mut total: u64 = 0; + for chunk in chunks { + if !is_valid_hash(&chunk.h) { + return Err(DomainError::validation_error( + "Invalid chunk hash format. Expected BLAKE3 (64 hex characters)", + )); + } + if chunk.s == 0 || chunk.s > CDC_MAX_CHUNK as u64 { + return Err(DomainError::validation_error(format!( + "Chunk size {} out of bounds (1 ..= {CDC_MAX_CHUNK})", + chunk.s + ))); + } + total = total.saturating_add(chunk.s); + } + if total > self.max_total_size { + return Err(DomainError::validation_error(format!( + "Declared total of {total} bytes exceeds the {}-byte upload ceiling", + self.max_total_size + ))); + } + Ok(total) + } + + /// Step 1: which of these chunks must the caller upload? + /// + /// Purely advisory and user-scoped — the commit re-checks entitlement + /// atomically, so a stale answer can never leak content. + pub async fn negotiate_with_perms( + &self, + caller_id: Uuid, + request: &DeltaNegotiateRequest, + ) -> Result { + self.validate_chunk_list(&request.chunks)?; + + let distinct = distinct_hashes(&request.chunks); + let claimable = self.dedup.claimable_chunks(caller_id, &distinct).await?; + let missing = distinct + .into_iter() + .filter(|h| !claimable.contains(h)) + .collect(); + Ok(DeltaNegotiateResponse { missing }) + } + + /// Step 2: store uploaded chunk frames. Hashes are computed + /// server-side; chunks land as unreferenced orphans awaiting a commit. + pub async fn receive_chunks(&self, frames: S) -> Result + where + S: Stream> + Send, + { + let received = self.dedup.store_loose_chunks(frames).await?; + Ok(DeltaChunksResponse { + received: received + .into_iter() + .map(|(h, s)| ChunkRef { h, s }) + .collect(), + }) + } + + /// Step 3: pin → verify → attach manifest → create/update the file row. + pub async fn commit_with_perms( + &self, + caller_id: Uuid, + request: DeltaCommitRequest, + ) -> Result { + // ── Shape ───────────────────────────────────────────────── + if !is_valid_hash(&request.file_hash) { + return Err(DomainError::validation_error( + "Invalid file_hash format. Expected BLAKE3 (64 hex characters)", + )); + } + let total_size = self.validate_chunk_list(&request.chunks)?; + + let mode = match (&request.name, &request.folder_id, &request.file_id) { + (Some(name), Some(folder_id), None) => { + let name = sanitize_file_name(name)?; + CommitMode::Create { + name, + folder_id: folder_id.clone(), + } + } + (None, None, Some(file_id)) => CommitMode::Update { + file_id: file_id.clone(), + }, + _ => { + return Err(DomainError::validation_error( + "Provide either name + folder_id (create) or file_id (update)", + )); + } + }; + + // ── AuthZ first: nothing is pinned for callers who may not write ── + match &mode { + CommitMode::Create { folder_id, .. } => { + let folder_uuid = Uuid::parse_str(folder_id) + .map_err(|_| DomainError::not_found("Folder", folder_id.clone()))?; + self.authz + .require( + Subject::User(caller_id), + Permission::Create, + Resource::Folder(folder_uuid), + ) + .await?; + } + CommitMode::Update { file_id } => { + let file_uuid = Uuid::parse_str(file_id) + .map_err(|_| DomainError::not_found("File", file_id.clone()))?; + self.authz + .require( + Subject::User(caller_id), + Permission::Update, + Resource::File(file_uuid), + ) + .await?; + } + } + + // ── Quota on the logical size (same semantics as a byte upload) ── + self.quota + .check_storage_quota(caller_id, total_size) + .await?; + + // ── Whole-file fast path: caller already owns this exact content ── + // Mirrors the instant-upload endpoint: a reference bump, no chunk + // work at all. Ownership is required — an existing-but-foreign + // manifest must be earned through the pin + verify path below. + if self + .dedup + .user_owns_blob_reference(&request.file_hash, &caller_id.to_string()) + .await + && let Some(metadata) = self.dedup.get_blob_metadata(&request.file_hash).await + { + self.dedup.add_reference(&request.file_hash).await?; + let blob = StoredBlob { + hash: request.file_hash.clone(), + size: metadata.size, + is_new_blob: false, + }; + let file = self + .register_row(caller_id, &mode, metadata.content_type, blob) + .await?; + return Ok(DeltaCommitOutcome::Done { + file, + created: matches!(mode, CommitMode::Create { .. }), + }); + } + + // ── Pin: atomically take one reference per distinct entitled chunk ── + let distinct = distinct_hashes(&request.chunks); + let pinned = self + .dedup + .pin_claimable_chunks(caller_id, &distinct) + .await?; + if pinned.len() != distinct.len() { + let still_missing: Vec = distinct + .iter() + .filter(|h| !pinned.contains(*h)) + .cloned() + .collect(); + let pinned_vec: Vec = pinned.into_iter().collect(); + self.dedup.release_pinned_chunks(&pinned_vec).await; + tracing::debug!( + "Delta commit: {} of {} chunks not claimable — client must upload them", + still_missing.len(), + distinct.len() + ); + return Ok(DeltaCommitOutcome::StillMissing(still_missing)); + } + + // ── Verify: the declared file_hash is recomputed from the pinned + // bytes before any manifest row can exist. ── + let verification = self + .dedup + .hash_chunk_sequence( + &request + .chunks + .iter() + .map(|c| (c.h.clone(), c.s)) + .collect::>(), + MAGIC_BYTES_LEN, + ) + .await; + let (computed_hash, head) = match verification { + Ok(v) => v, + Err(e) => { + self.dedup.release_pinned_chunks(&distinct).await; + tracing::info!( + target: "audit", + event = "delta_upload.rejected", + reason = "chunk_verification_failed", + caller_id = %caller_id, + file_hash = %request.file_hash, + "👮🏻‍♂️ Delta commit rejected: chunk sequence failed verification read", + ); + return Err(e); + } + }; + if computed_hash != request.file_hash { + self.dedup.release_pinned_chunks(&distinct).await; + tracing::info!( + target: "audit", + event = "delta_upload.rejected", + reason = "file_hash_mismatch", + caller_id = %caller_id, + declared_hash = %request.file_hash, + computed_hash = %computed_hash, + "👮🏻‍♂️ Delta commit rejected: declared file_hash does not match the chunk sequence", + ); + return Err(DomainError::validation_error( + "file_hash does not match the chunk sequence", + )); + } + + // ── Attach the manifest (shared accounting with the byte path) ── + let display_name = match &mode { + CommitMode::Create { name, .. } => name.clone(), + CommitMode::Update { file_id } => file_id.clone(), + }; + let content_type = match refine_content_type(&head, &display_name, "") { + ct if ct.is_empty() => "application/octet-stream".to_string(), + ct => ct, + }; + let chunk_hashes: Vec = request.chunks.iter().map(|c| c.h.clone()).collect(); + let chunk_sizes: Vec = request.chunks.iter().map(|c| c.s).collect(); + let attached = self + .dedup + .attach_manifest( + &request.file_hash, + &chunk_hashes, + &chunk_sizes, + total_size, + Some(content_type.clone()), + &distinct, + ) + .await?; + + let blob = StoredBlob { + hash: request.file_hash.clone(), + size: attached.size(), + is_new_blob: !attached.was_deduplicated(), + }; + let file = self + .register_row(caller_id, &mode, Some(content_type), blob) + .await?; + Ok(DeltaCommitOutcome::Done { + file, + created: matches!(mode, CommitMode::Create { .. }), + }) + } + + // ── Delta download ("download only what changed") ──────────── + + /// The chunk recipe of a file the caller owns — step 1 of a delta + /// download. Owner-scoped like the rest of the delta protocol (shared + /// files use the regular download endpoints); a non-owned or unknown + /// file is `NotFound`, and the denial is audited. + pub async fn file_manifest_with_perms( + &self, + caller_id: Uuid, + file_id: &str, + ) -> Result { + let file_uuid = Uuid::parse_str(file_id) + .map_err(|_| DomainError::not_found("File", file_id.to_string()))?; + // Engine check first (Read on the file): owners always pass and + // the engine audits denials; the ownership constraint below is the + // chunk layer's own entitlement standard. + self.authz + .require( + Subject::User(caller_id), + Permission::Read, + Resource::File(file_uuid), + ) + .await?; + + let file = self.file_read.get_file(file_id).await?; + let file_hash = file.content_hash().to_string(); + if !self + .dedup + .user_owns_blob_reference(&file_hash, &caller_id.to_string()) + .await + { + // Read permission without ownership (e.g. a grant): the delta + // surface is owner-scoped — the regular download endpoints + // serve shared content. + tracing::info!( + target: "audit", + event = "delta_download.rejected", + reason = "manifest_not_owner", + caller_id = %caller_id, + file_id = %file_id, + "👮🏻‍♂️ Delta download rejected: manifest requested by a non-owner", + ); + return Err(DomainError::not_found("File", file_id.to_string())); + } + + let Some((chunks, total_size)) = self.dedup.manifest_chunk_list(&file_hash).await? else { + return Err(DomainError::not_found("File", file_id.to_string())); + }; + Ok(DeltaManifestResponse { + file_hash, + total_size, + chunks: chunks.into_iter().map(|(h, s)| ChunkRef { h, s }).collect(), + }) + } + + /// Authorize a batched chunk download — step 2. Every hash must be + /// reachable through the caller's own files; otherwise the full list of + /// unavailable hashes is returned (the same information N individual + /// requests would reveal, in one round-trip). The total payload is + /// bounded by the per-request budget so clients split large deltas. + pub async fn authorize_chunk_download_with_perms( + &self, + caller_id: Uuid, + request: &DeltaDownloadRequest, + ) -> Result { + if request.hashes.is_empty() { + return Err(DomainError::validation_error("hashes must not be empty")); + } + if request.hashes.len() > self.max_chunk_count() { + return Err(DomainError::validation_error(format!( + "Too many chunks: {} (maximum {})", + request.hashes.len(), + self.max_chunk_count() + ))); + } + let mut distinct_seen = HashSet::new(); + for hash in &request.hashes { + if !is_valid_hash(hash) { + return Err(DomainError::validation_error( + "Invalid chunk hash format. Expected BLAKE3 (64 hex characters)", + )); + } + if !distinct_seen.insert(hash.as_str()) { + return Err(DomainError::validation_error( + "Duplicate hashes in download request", + )); + } + } + + let entitled = self + .dedup + .claimable_chunks(caller_id, &request.hashes) + .await?; + let not_available: Vec = request + .hashes + .iter() + .filter(|h| !entitled.contains(*h)) + .cloned() + .collect(); + if !not_available.is_empty() { + tracing::info!( + target: "audit", + event = "delta_download.rejected", + reason = "chunks_not_owned", + caller_id = %caller_id, + requested = request.hashes.len(), + denied = not_available.len(), + "👮🏻‍♂️ Delta download rejected: caller requested chunks outside their files", + ); + return Ok(DeltaDownloadOutcome::NotAvailable(not_available)); + } + + let sizes = self.dedup.chunk_sizes(&request.hashes).await?; + let mut ordered = Vec::with_capacity(request.hashes.len()); + let mut total: u64 = 0; + for hash in &request.hashes { + // Entitled implies indexed; a vanished row between the two + // queries surfaces as unavailable rather than a 500. + let Some(size) = sizes.get(hash) else { + return Ok(DeltaDownloadOutcome::NotAvailable(vec![hash.clone()])); + }; + total = total.saturating_add(*size); + ordered.push((hash.clone(), *size)); + } + if total > self.max_download_batch { + return Err(DomainError::validation_error(format!( + "Requested chunks total {total} bytes; the per-request ceiling is {} — split the download into smaller batches", + self.max_download_batch + ))); + } + Ok(DeltaDownloadOutcome::Ready(ordered)) + } + + /// Stream one authorized chunk's bytes (entitlement was established by + /// [`authorize_chunk_download_with_perms`]). + pub async fn chunk_stream( + &self, + hash: &str, + ) -> Result< + std::pin::Pin> + Send>>, + DomainError, + > { + self.dedup.chunk_stream(hash).await + } + + /// Create or update the file row against a blob reference the commit + /// already holds (the registration paths release it on failure). + async fn register_row( + &self, + caller_id: Uuid, + mode: &CommitMode, + content_type: Option, + blob: StoredBlob, + ) -> Result { + match mode { + CommitMode::Create { name, folder_id } => { + let content_type = + content_type.unwrap_or_else(|| "application/octet-stream".to_string()); + self.uploads + .upload_file_streaming( + name.clone(), + Some(folder_id.clone()), + content_type, + blob, + ) + .await + } + CommitMode::Update { file_id } => { + self.uploads + .update_file_content_by_id_with_perms(caller_id, file_id, blob) + .await + } + } + } +} + +/// 64 lowercase/uppercase hex characters. +fn is_valid_hash(hash: &str) -> bool { + hash.len() == 64 && hash.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// Basename only — same path-traversal guard as the upload handlers. +fn sanitize_file_name(name: &str) -> Result { + let base = name + .rsplit('/') + .next() + .unwrap_or(name) + .rsplit('\\') + .next() + .unwrap_or(name) + .trim(); + if base.is_empty() { + return Err(DomainError::validation_error("File name must not be empty")); + } + Ok(base.to_string()) +} + +/// Distinct hashes in first-occurrence order. +fn distinct_hashes(chunks: &[ChunkRef]) -> Vec { + let mut seen = HashSet::new(); + chunks + .iter() + .filter(|c| seen.insert(c.h.as_str())) + .map(|c| c.h.clone()) + .collect() +} diff --git a/src/application/services/file_upload_service.rs b/src/application/services/file_upload_service.rs index 92ec9215..fabcb5c1 100644 --- a/src/application/services/file_upload_service.rs +++ b/src/application/services/file_upload_service.rs @@ -1,16 +1,19 @@ -use std::path::{Path, PathBuf}; use std::sync::Arc; +use uuid::Uuid; use crate::application::dtos::file_dto::FileDto; +use crate::application::ports::authorization_ports::AuthorizationEngine; use crate::application::ports::file_lifecycle::FileLifecycleHook; -use crate::application::ports::file_ports::FileUploadUseCase; -use crate::application::ports::storage_ports::{FileReadPort, FileWritePort}; +use crate::application::ports::file_ports::{FileUploadUseCase, StoredBlob}; +use crate::application::ports::storage_ports::{FileReadPort, FileWritePort, StorageUsagePort}; use crate::application::services::storage_usage_service::StorageUsageService; use crate::common::errors::DomainError; +use crate::domain::services::authorization::{Permission, Resource, Subject}; use crate::infrastructure::repositories::pg::FileBlobReadRepository; use crate::infrastructure::repositories::pg::FileBlobWriteRepository; use crate::infrastructure::services::dedup_service::DedupService; use crate::infrastructure::services::file_content_cache::FileContentCache; +use crate::infrastructure::services::pg_acl_engine::PgAclEngine; use tracing::{debug, info, warn}; /// Helper function to extract username from folder path string. @@ -34,20 +37,16 @@ fn extract_username_from_path(path: &str) -> Option { /// Service for file upload operations. /// -/// **Every upload path converges on streaming-to-disk** — there is no -/// `Vec` buffer path. -/// -/// - **Normal uploads**: handler spools multipart to temp file → `upload_file_streaming` -/// - **Chunked uploads**: chunks already on disk → `upload_file_from_path` -/// - **WebDAV PUT (large)**: handler streams body to temp file → `update_file_streaming` -/// - **WebDAV PUT (small / compat)**: `create_file` / `update_file` spool `&[u8]` -/// to a temp file internally, then call the streaming path. -/// -/// Peak RAM usage during any upload: ~256 KB (streaming hash) regardless of file size. +/// Content never passes through this service: the interface layer streams +/// the request body straight into the CDC chunk store (no spool file, no +/// full-body buffering) and hands over a [`StoredBlob`] reference. This +/// service registers the metadata row, keeps caches coherent and fires +/// lifecycle hooks. Blob-reference ownership is consumed by the write +/// port, which releases it on failure — callers never compensate. pub struct FileUploadService { - /// Write port — handles save, streaming, deferred registration + /// Write port — registers file rows against ingested blobs. file_write: Arc, - /// Read port — needed for WebDAV create_file / update_file + /// Read port — needed for WebDAV/WOPI update-by-path. file_read: Option>, /// Optional storage usage tracking storage_usage_service: Option>, @@ -55,9 +54,18 @@ pub struct FileUploadService { content_cache: Option>, /// Single lifecycle dispatcher — fires on_file_created / on_file_updated. file_lifecycle_hook: Option>, - /// Directory for spool temp files (`&[u8]` upload variants). When `Some`, - /// keeps spools off tmpfs/RAM so they don't count against the cgroup limit. - upload_temp_dir: Option, + /// Dependencies of the instant-upload path + /// (`create_file_from_owned_blob_with_perms`); `None` in minimal test + /// wiring. + instant_upload: Option, +} + +/// Everything the instant-upload path needs beyond the upload service's own +/// ports: permission checks, the dedup index, and quota enforcement. +struct InstantUploadDeps { + authz: Arc, + dedup: Arc, + quota: Arc, } impl FileUploadService { @@ -69,7 +77,7 @@ impl FileUploadService { storage_usage_service: None, content_cache: None, file_lifecycle_hook: None, - upload_temp_dir: None, + instant_upload: None, } } @@ -84,13 +92,23 @@ impl FileUploadService { storage_usage_service: None, content_cache: None, file_lifecycle_hook: None, - upload_temp_dir: None, + instant_upload: None, } } - /// Configures the spool directory for the `&[u8]` upload variants. - pub fn with_upload_temp_dir(mut self, dir: Option) -> Self { - self.upload_temp_dir = dir; + /// Wires the authorization engine, dedup index and quota service that + /// power the instant-upload path. + pub fn with_instant_upload( + mut self, + authz: Arc, + dedup: Arc, + quota: Arc, + ) -> Self { + self.instant_upload = Some(InstantUploadDeps { + authz, + dedup, + quota, + }); self } @@ -115,13 +133,178 @@ impl FileUploadService { self } - // ── private helpers ────────────────────────────────────────── + // ── Instant upload (zero content bytes) ────────────────────── - /// Create a spool temp file, honoring the configured upload temp dir. - fn new_temp(&self) -> std::io::Result { - crate::common::temp::new_spool_temp_file(self.upload_temp_dir.as_deref()) + /// Register a new file row pointing at a blob the caller **already + /// owns** — the instant-upload path: the client proved it has the + /// content by hash, so no bytes travel and no chunk is written. Pure + /// metadata: one ref_count bump + one row INSERT. + /// + /// Security model (mirrors `GET /api/dedup/check/{hash}`): + /// - The caller must have `Create` permission on the target folder. + /// - The hash is only claimable when the caller owns at least one + /// non-trashed file referencing it — never a global content oracle. + /// A non-owned hash returns `NotFound` (anti-enumeration: same shape + /// as "no such blob") and emits an `instant_upload.rejected` audit + /// event with the real reason. + /// - Quota is enforced on the logical size, exactly like a byte upload. + pub async fn create_file_from_owned_blob_with_perms( + &self, + caller_id: Uuid, + name: String, + folder_id: String, + hash: &str, + ) -> Result { + let Some(InstantUploadDeps { + authz, + dedup, + quota, + }) = &self.instant_upload + else { + return Err(DomainError::internal_error( + "FileUpload", + "instant upload is not wired (authz/dedup/quota missing)", + )); + }; + + // ── AuthZ: Create on the target folder ─────────────────── + let folder_uuid = Uuid::parse_str(&folder_id) + .map_err(|_| DomainError::not_found("Folder", folder_id.clone()))?; + authz + .require( + Subject::User(caller_id), + Permission::Create, + Resource::Folder(folder_uuid), + ) + .await?; + + // ── Ownership: only blobs the caller can already read ──── + if !dedup + .user_owns_blob_reference(hash, &caller_id.to_string()) + .await + { + tracing::info!( + target: "audit", + event = "instant_upload.rejected", + reason = "hash_not_owned", + caller_id = %caller_id, + blob_hash = %hash, + "👮🏻‍♂️ Instant upload rejected: caller owns no file referencing the claimed hash", + ); + return Err(DomainError::not_found("Blob", hash)); + } + + let Some(metadata) = dedup.get_blob_metadata(hash).await else { + // Lost a race with the last-reference delete — same shape as + // "never existed". + return Err(DomainError::not_found("Blob", hash)); + }; + + // ── Quota on the logical size, before taking any reference ── + quota.check_storage_quota(caller_id, metadata.size).await?; + + // The manifest knows the original content type; fall back to the + // new name's extension when the stored one is generic. + let claimed = metadata.content_type.as_deref().unwrap_or(""); + let content_type = + match crate::common::mime_detect::refine_content_type(&[], &name, claimed) { + ct if ct.is_empty() => "application/octet-stream".to_string(), + ct => ct, + }; + + // Take the reference the row registration will consume (it releases + // it again on any failure). A concurrent GC between the ownership + // check and this bump surfaces as NotFound — the client falls back + // to a normal byte upload. + dedup.add_reference(hash).await?; + + let dto = self + .upload_file_streaming( + name, + Some(folder_id), + content_type, + StoredBlob { + hash: hash.to_string(), + size: metadata.size, + is_new_blob: false, + }, + ) + .await?; + + info!( + "⚡ INSTANT UPLOAD: {} ({} bytes, 0 transferred, ID: {})", + dto.name, metadata.size, dto.id + ); + Ok(dto) } + /// Swap an existing file's content to an already-ingested blob — the + /// update mode of the delta-upload commit. The caller needs `Write` + /// permission on the file; the blob reference is consumed (released on + /// failure by the write port, like every other registration path). + pub async fn update_file_content_by_id_with_perms( + &self, + caller_id: Uuid, + file_id: &str, + blob: StoredBlob, + ) -> Result { + let Some(InstantUploadDeps { authz, .. }) = &self.instant_upload else { + return Err(DomainError::internal_error( + "FileUpload", + "instant upload is not wired (authz/dedup/quota missing)", + )); + }; + let Some(file_read) = &self.file_read else { + return Err(DomainError::internal_error( + "FileUpload", + "read port is not wired", + )); + }; + + let file_uuid = Uuid::parse_str(file_id) + .map_err(|_| DomainError::not_found("File", file_id.to_string()))?; + authz + .require( + Subject::User(caller_id), + Permission::Update, + Resource::File(file_uuid), + ) + .await?; + + let file = file_read.get_file(file_id).await?; + let (new_hash, updated_at) = self + .file_write + .update_file_content_with_blob(file_id, &blob.hash, blob.size, None) + .await?; + // The file maps to a different blob now — stale cached content must + // never be served for the rest of its TTI window. + if let Some(cc) = &self.content_cache { + cc.invalidate(file_id).await; + } + + let parts = file.into_parts(); + let updated = crate::domain::entities::file::File::with_timestamps_and_blob_hash( + parts.id, + parts.name, + parts.storage_path, + blob.size, + parts.mime_type, + parts.folder_id, + parts.created_at, + updated_at as u64, + parts.owner_id, + new_hash, + ) + .map_err(|e| DomainError::internal_error("FileUpload", format!("rebuild entity: {e}")))?; + let dto = FileDto::from(updated); + if let Some(hook) = &self.file_lifecycle_hook { + hook.on_file_updated(file_id, &dto.content_hash, &dto.mime_type); + } + Ok(dto) + } + + // ── private helpers ────────────────────────────────────────── + /// Optionally update storage usage after a successful upload. fn maybe_update_storage_usage(&self, file: &FileDto) { if let Some(storage_service) = &self.storage_usage_service { @@ -146,172 +329,37 @@ impl FileUploadService { } impl FileUploadUseCase for FileUploadService { - /// Streaming upload from a temp file on disk. - /// - /// Peak RAM: ~256 KB (hash calculation) regardless of file size. - /// The temp file is consumed (moved/deleted) by the blob store. + /// Register a new file row pointing at an already-ingested blob. async fn upload_file_streaming( &self, name: String, folder_id: Option, content_type: String, - temp_path: &Path, - size: u64, - pre_computed_hash: Option, + blob: StoredBlob, ) -> Result { - let (file, is_new_blob) = self + let file = self .file_write - .save_file_from_temp_with_dedup( - name.clone(), - folder_id, - content_type, - temp_path, - size, - pre_computed_hash, - ) + .save_file_with_blob(name.clone(), folder_id, content_type, &blob.hash, blob.size) .await?; let dto = FileDto::from(file); info!( "📡 STREAMING UPLOAD: {} ({} bytes, ID: {})", - name, size, dto.id + name, blob.size, dto.id ); self.maybe_update_storage_usage(&dto); if let Some(hook) = &self.file_lifecycle_hook { - hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, is_new_blob); + hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, blob.is_new_blob); } Ok(dto) } - /// Upload from a file already on disk (chunked uploads). - async fn upload_file_from_path( - &self, - name: String, - folder_id: Option, - content_type: String, - file_path: &Path, - pre_computed_hash: Option, - ) -> Result { - let size = tokio::fs::metadata(file_path) - .await - .map_err(|e| { - DomainError::internal_error( - "FileUpload", - format!("Failed to read file metadata: {}", e), - ) - })? - .len(); - - self.upload_file_streaming( - name, - folder_id, - content_type, - file_path, - size, - pre_computed_hash, - ) - .await - } - - /// Creates a file at a specific path (for WebDAV PUT on new resource). - /// - /// Spools the in-memory `&[u8]` to a temp file with hash-on-write, - /// then delegates to the streaming path. Peak RAM: the caller's - /// buffer + ~256 KB for the hasher. - async fn create_file( - &self, - parent_path: &str, - filename: &str, - content: &[u8], - content_type: &str, - ) -> Result { - // Look up the folder ID by folder path - let parent_id = if !parent_path.is_empty() { - if let Some(file_read) = &self.file_read { - file_read.get_folder_id_by_path(parent_path).await.ok() - } else { - None - } - } else { - None - }; - - // Spool to temp file + hash - let temp = self - .new_temp() - .map_err(|e| DomainError::internal_error("FileUpload", format!("temp file: {e}")))?; - tokio::fs::write(temp.path(), content) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?; - let hash = DedupService::hash_file(temp.path()) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("hash: {e}")))?; - - let (file, is_new_blob) = self - .file_write - .save_file_from_temp_with_dedup( - filename.to_string(), - parent_id, - content_type.to_string(), - temp.path(), - content.len() as u64, - Some(hash), - ) - .await?; - let dto = FileDto::from(file); - self.maybe_update_storage_usage(&dto); - if let Some(hook) = &self.file_lifecycle_hook { - hook.on_file_created(&dto.id, &dto.content_hash, &dto.mime_type, is_new_blob); - } - Ok(dto) - } - - /// Updates an existing file's content, or creates it if not found (for WebDAV PUT). - /// - /// Spools the in-memory `&[u8]` to a temp file with hash-on-write, - /// then delegates to the streaming update/create path. - async fn update_file( - &self, - path: &str, - content: &[u8], - content_type: &str, - modified_at: Option, - ) -> Result { - // Spool to temp file + hash - let temp = self - .new_temp() - .map_err(|e| DomainError::internal_error("FileUpload", format!("temp file: {e}")))?; - tokio::fs::write(temp.path(), content) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("write temp: {e}")))?; - let hash = DedupService::hash_file(temp.path()) - .await - .map_err(|e| DomainError::internal_error("FileUpload", format!("hash: {e}")))?; - - self.update_file_streaming( - path, - temp.path(), - content.len() as u64, - content_type, - Some(hash), - modified_at, - ) - .await - } - - /// Streaming update — replaces file content from a temp file on disk. - /// - /// Uses `update_file_content_from_temp` which passes the pre-computed hash - /// to dedup, avoiding a second full read of the file. - /// For new files (not found at `path`), falls back to `upload_file_streaming`. - /// - /// Peak RAM: ~256 KB regardless of file size. + /// Swap the content of the file at `path` to an already-ingested blob, + /// creating the file when it doesn't exist (WebDAV/NextCloud/WOPI PUT). async fn update_file_streaming( &self, path: &str, - temp_path: &Path, - size: u64, + blob: StoredBlob, content_type: &str, - pre_computed_hash: Option, modified_at: Option, ) -> Result { // Try to find the existing file first @@ -321,14 +369,7 @@ impl FileUploadUseCase for FileUploadService { let file_id = file.id().to_string(); let (new_hash, updated_at) = self .file_write - .update_file_content_from_temp( - &file_id, - temp_path, - size, - Some(content_type.to_string()), - pre_computed_hash, - modified_at, - ) + .update_file_content_with_blob(&file_id, &blob.hash, blob.size, modified_at) .await?; // Invalidate content cache — file content has changed. if let Some(cc) = &self.content_cache { @@ -343,7 +384,7 @@ impl FileUploadUseCase for FileUploadService { parts.id, parts.name, parts.storage_path, - size, + blob.size, parts.mime_type, parts.folder_id, parts.created_at, @@ -381,15 +422,15 @@ impl FileUploadUseCase for FileUploadService { None }; - let (created, is_new_blob) = self + let is_new_blob = blob.is_new_blob; + let created = self .file_write - .save_file_from_temp_with_dedup( + .save_file_with_blob( filename.to_string(), parent_id, content_type.to_string(), - temp_path, - size, - pre_computed_hash, + &blob.hash, + blob.size, ) .await?; let dto = FileDto::from(created); diff --git a/src/application/services/idor_protection_test.rs b/src/application/services/idor_protection_test.rs index 8682ae28..33d1dcf7 100644 --- a/src/application/services/idor_protection_test.rs +++ b/src/application/services/idor_protection_test.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use futures::Stream; use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::pin::Pin; use std::sync::Mutex; use uuid::Uuid; @@ -169,14 +169,13 @@ impl MockFileWritePort { } impl FileWritePort for MockFileWritePort { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> Result { unimplemented!() } @@ -205,13 +204,11 @@ impl FileWritePort for MockFileWritePort { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result<(String, i64), DomainError> { Ok((String::new(), 0)) diff --git a/src/application/services/mod.rs b/src/application/services/mod.rs index 7fa82755..7006ef9f 100644 --- a/src/application/services/mod.rs +++ b/src/application/services/mod.rs @@ -5,6 +5,7 @@ pub mod batch_operations; pub mod blob_lifecycle_service; pub mod calendar_service; pub mod contact_service; +pub mod delta_upload_service; pub mod device_auth_service; pub mod external_identity_service; pub mod favorites_service; diff --git a/src/application/services/trash_service_test.rs b/src/application/services/trash_service_test.rs index 951abf39..675bb300 100644 --- a/src/application/services/trash_service_test.rs +++ b/src/application/services/trash_service_test.rs @@ -554,14 +554,13 @@ impl FileReadPort for MockFileRepository { } impl FileWritePort for MockFileRepository { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &std::path::Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> std::result::Result { unimplemented!() } @@ -586,13 +585,11 @@ impl FileWritePort for MockFileRepository { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &std::path::Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> std::result::Result<(String, i64), DomainError> { Ok((String::new(), 0)) diff --git a/src/common/config.rs b/src/common/config.rs index 963454e2..1c7e0a93 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -228,12 +228,6 @@ pub struct StorageConfig { /// returns 413 with a "use chunked upload" hint when a direct PUT /// exceeds this cap. Env: `OXICLOUD_DIRECT_PUT_MAX_BYTES`. pub direct_put_max_bytes: usize, - /// Directory for upload spool temp files. When `Some`, large uploads are - /// spooled here instead of the OS default temp dir (often tmpfs/RAM in - /// containers, where the spool's page-cache counts against the cgroup - /// memory limit and can trigger OOMKill on large files). Env: - /// `OXICLOUD_UPLOAD_TMPDIR`. - pub upload_temp_dir: Option, /// Root directory for chunked-upload sessions. When `Some`, chunks land /// under `{chunk_dir}/{upload_id}/` (REST) and /// `{chunk_dir}/nextcloud/{user}/{upload_id}/` (NC). When `None`, falls @@ -401,7 +395,6 @@ impl Default for StorageConfig { max_upload_size: MAX_UPLOAD_SIZE, chunk_max_bytes: 100 * 1024 * 1024, // 100 MB — sane upper bound for a single chunked-upload PUT direct_put_max_bytes: 1024 * 1024 * 1024, // 1 GiB — pushes larger uploads onto the chunked protocol - upload_temp_dir: None, chunk_dir: None, usage_reconcile_secs: 600, // 10 minutes tree_etag_flush_ms: 500, @@ -1349,18 +1342,10 @@ impl AppConfig { config.storage.direct_put_max_bytes = val; } - // Upload spool directory — keep large upload temp files off tmpfs/RAM - // (otherwise their page-cache counts against the cgroup memory limit). - if let Ok(dir) = env::var("OXICLOUD_UPLOAD_TMPDIR") - && !dir.trim().is_empty() - { - config.storage.upload_temp_dir = Some(PathBuf::from(dir.trim())); - } - // Chunked-upload session root — separate from the PUT spool because - // chunked sessions accumulate disk on long uploads (multi-chunk - // resumable transfers) while PUT spool is short-lived. Sysadmins - // commonly want one of them on fast/local storage (NVMe) and the - // other on bulk storage; this knob lets that be expressed. + // Chunked-upload session root — chunked sessions accumulate disk on + // long uploads (multi-chunk resumable transfers); sysadmins commonly + // want them on fast/local storage (NVMe). This knob lets that be + // expressed. if let Ok(dir) = env::var("OXICLOUD_CHUNK_DIR") && !dir.trim().is_empty() { diff --git a/src/common/di.rs b/src/common/di.rs index 0c9fcb19..ee4dafc5 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -441,6 +441,7 @@ impl AppServiceFactory { repos: &RepositoryServices, trash_service: Option>, authz: &Arc, + storage_usage: &Arc, content_index: Option>, ) -> ApplicationServices { // Main services @@ -456,7 +457,25 @@ impl AppServiceFactory { ) .with_content_cache(core.file_content_cache.clone()) .with_file_lifecycle_hook(core.file_lifecycle.clone()) - .with_upload_temp_dir(self.config.storage.upload_temp_dir.clone()), + .with_instant_upload( + authz.clone(), + core.dedup_service.clone(), + storage_usage.clone(), + ), + ); + + // Delta-upload protocol — chunk negotiation over the same dedup + // store. Bounded by the same whole-file ceiling as byte uploads. + let delta_upload_service = Arc::new( + crate::application::services::delta_upload_service::DeltaUploadService::new( + core.dedup_service.clone(), + file_upload_service.clone(), + repos.file_read_repository.clone(), + storage_usage.clone(), + authz.clone(), + self.config.storage.max_upload_size as u64, + self.config.storage.chunk_max_bytes as u64, + ), ); let file_retrieval_service = Arc::new(FileRetrievalService::new_with_cache( @@ -508,6 +527,7 @@ impl AppServiceFactory { // Traits for abstraction folder_service, file_upload_service, + delta_upload_service, file_retrieval_service, file_management_service, file_use_case_factory, @@ -566,9 +586,12 @@ impl AppServiceFactory { .with_file_deleted_hook(core.file_lifecycle.clone()), ); - // Initialize cleanup service (bulk-deletes expired items in 2 SQL queries) + // Initialize cleanup service (bulk-deletes expired items in 2 SQL + // queries, then GCs zero-reference blobs — including chunks orphaned + // by aborted streaming uploads). let cleanup_service = TrashCleanupService::new( trash_repo.clone(), + core.dedup_service.clone(), 24, // Run cleanup every 24 hours ); @@ -799,7 +822,12 @@ impl AppServiceFactory { .create_trash_service(&repos, &core, &authorization) .await; - // 3c. Content index (embedded Tantivy) — opened before application + // 3c. Storage usage / quota service (needed by the instant-upload + // path inside the application services, and re-exposed on AppState + // for the handler-side quota checks of the byte-upload paths). + let storage_usage = self.create_storage_usage_service(&repos, &pool, &maintenance_pool); + + // 3d. Content index (embedded Tantivy) — opened before application // services so SearchService can hold the query port; the feeding // worker starts further down with the maintenance pool. let content_index = self.create_content_index(); @@ -810,6 +838,7 @@ impl AppServiceFactory { &repos, trash_service.clone(), &authorization, + &storage_usage, content_index.as_ref().map(|(idx, _)| idx.clone()), ); @@ -851,8 +880,7 @@ impl AppServiceFactory { recent_service = Some(recent.clone()); apps.recent_service = Some(recent); - storage_usage_service = - Some(self.create_storage_usage_service(&repos, &pool, &maintenance_pool)); + storage_usage_service = Some(storage_usage.clone()); self.start_tree_etag_flush_job(&maintenance_pool); @@ -1103,6 +1131,13 @@ impl AppServiceFactory { user_profile_rate_limiter: Arc::new( crate::interfaces::middleware::rate_limit::RateLimiter::new(60, 60, 50_000), ), + // Delta upload: 240 requests / minute / caller. Generous for a + // real client (chunk PUTs carry up to 100 MB each) while + // stopping pin/negotiate floods; 50 000 tracked callers bound + // the memory like the other limiters. + delta_upload_rate_limiter: Arc::new( + crate::interfaces::middleware::rate_limit::RateLimiter::new(240, 60, 50_000), + ), // PR 12 — per-sharer email-invite ceiling: caller_id-keyed. // Defends against a compromised account spamming external // invites (each invite mints a new external user + email). @@ -1440,6 +1475,8 @@ pub struct ApplicationServices { // Traits for abstraction pub folder_service: Arc, pub file_upload_service: Arc, + pub delta_upload_service: + Arc, pub file_retrieval_service: Arc, pub file_management_service: Arc, pub file_use_case_factory: Arc, @@ -1559,6 +1596,9 @@ pub struct AppState { /// authenticated caller covers any legitimate UI rendering while /// throttling enumeration. pub user_profile_rate_limiter: Arc, + /// Per-caller flood guard for the delta-upload endpoints + /// (negotiate / chunks / commit share one budget). + pub delta_upload_rate_limiter: Arc, /// Per-sharer ceiling on `POST /api/grants` invitations whose /// subject is `{ type: "email" }`. 50 per hour keyed on /// `caller_id`. Anonymous attackers can't reach this code path diff --git a/src/common/mime_detect.rs b/src/common/mime_detect.rs index 8cecd2c9..d2fcf118 100644 --- a/src/common/mime_detect.rs +++ b/src/common/mime_detect.rs @@ -8,21 +8,25 @@ //! //! Performance: < 1µs for the `infer` check (reads only header bytes, no allocation). -use std::path::Path; -use tokio::io::AsyncReadExt; - -/// Maximum bytes to read for magic-byte detection. -const MAGIC_BYTES_LEN: usize = 8192; +/// Maximum bytes needed for magic-byte detection. Upload ingestion peeks +/// this many bytes off the stream before forwarding them unchanged. +pub const MAGIC_BYTES_LEN: usize = 8192; /// Extract the filename component from a `/`-separated path. pub fn filename_from_path(path: &str) -> &str { path.rsplit('/').next().unwrap_or(path) } +/// Whether a claimed Content-Type is too generic to trust — these trigger +/// magic-byte detection on the upload path. +pub fn is_generic_mime(claimed: &str) -> bool { + claimed.is_empty() || claimed == "application/octet-stream" || claimed == "binary/octet-stream" +} + /// Refine a claimed MIME type using magic bytes and filename extension. /// /// This is a synchronous function — the caller should already have the first -/// bytes of the file available (or call the async wrapper below). +/// bytes of the content available (upload ingestion peeks them in-flight). /// /// # Arguments /// * `buf` — first bytes of the file (at least 8192 for best results) @@ -30,10 +34,7 @@ pub fn filename_from_path(path: &str) -> &str { /// * `claimed` — the Content-Type sent by the client pub fn refine_content_type(buf: &[u8], filename: &str, claimed: &str) -> String { // If the client sent a specific type (not generic), trust it - if !claimed.is_empty() - && claimed != "application/octet-stream" - && claimed != "binary/octet-stream" - { + if !is_generic_mime(claimed) { return claimed.to_string(); } @@ -52,49 +53,9 @@ pub fn refine_content_type(buf: &[u8], filename: &str, claimed: &str) -> String claimed.to_string() } -/// Async helper: reads the first bytes of a file on disk and refines the MIME type. -/// -/// Designed for the upload path where the file has been spooled to a temp path. -pub async fn refine_content_type_from_file( - temp_path: &Path, - filename: &str, - claimed: &str, -) -> String { - // Fast path: if the client gave us a specific type, trust it - if !claimed.is_empty() - && claimed != "application/octet-stream" - && claimed != "binary/octet-stream" - { - return claimed.to_string(); - } - - // Read only the first bytes needed for magic detection (not the whole file). - match tokio::fs::File::open(temp_path).await { - Ok(mut file) => { - let mut buf = vec![0u8; MAGIC_BYTES_LEN]; - let n = file.read(&mut buf).await.unwrap_or(0); - refine_content_type(&buf[..n], filename, claimed) - } - Err(e) => { - tracing::warn!( - "MIME detection: failed to read {} for magic bytes: {}", - temp_path.display(), - e - ); - // Fall back to extension - let guess = mime_guess::from_path(filename); - if let Some(mime) = guess.first() { - return mime.to_string(); - } - claimed.to_string() - } - } -} - #[cfg(test)] mod tests { use super::*; - use std::io::Write; // ── refine_content_type (sync) ────────────────────────────── @@ -145,47 +106,15 @@ mod tests { assert_eq!(result, "image/png"); } - // ── refine_content_type_from_file (async) ─────────────────── + // ── is_generic_mime ───────────────────────────────────────── - #[tokio::test] - async fn from_file_detects_png() { - let mut tmp = tempfile::NamedTempFile::new().unwrap(); - let png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"; - tmp.write_all(png).unwrap(); - tmp.flush().unwrap(); - - let result = - refine_content_type_from_file(tmp.path(), "photo", "application/octet-stream").await; - assert_eq!(result, "image/png"); - } - - #[tokio::test] - async fn from_file_falls_back_to_extension() { - let mut tmp = tempfile::NamedTempFile::new().unwrap(); - tmp.write_all(b"not magic").unwrap(); - tmp.flush().unwrap(); - - let result = - refine_content_type_from_file(tmp.path(), "doc.css", "application/octet-stream").await; - assert_eq!(result, "text/css"); - } - - #[tokio::test] - async fn from_file_trusts_specific_claimed() { - let result = - refine_content_type_from_file(Path::new("/nonexistent"), "file", "image/webp").await; - assert_eq!(result, "image/webp"); - } - - #[tokio::test] - async fn from_file_missing_file_falls_back_to_extension() { - let result = refine_content_type_from_file( - Path::new("/nonexistent/file"), - "photo.jpg", - "application/octet-stream", - ) - .await; - assert_eq!(result, "image/jpeg"); + #[test] + fn generic_mime_detection() { + assert!(is_generic_mime("")); + assert!(is_generic_mime("application/octet-stream")); + assert!(is_generic_mime("binary/octet-stream")); + assert!(!is_generic_mime("image/png")); + assert!(!is_generic_mime("text/plain")); } // ── filename_from_path ────────────────────────────────────── diff --git a/src/common/mod.rs b/src/common/mod.rs index ddb9e2c6..b343f797 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -4,4 +4,3 @@ pub mod errors; pub mod locale; pub mod mime_detect; pub mod stubs; -pub mod temp; diff --git a/src/common/stubs.rs b/src/common/stubs.rs index c2c3957e..e857085a 100644 --- a/src/common/stubs.rs +++ b/src/common/stubs.rs @@ -24,6 +24,7 @@ use crate::application::dtos::search_dto::{ }; use crate::application::ports::file_ports::{ FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, OptimizedFileContent, + StoredBlob, }; use crate::application::ports::folder_ports::FolderUseCase; @@ -149,14 +150,13 @@ impl FileReadPort for StubFileReadPort { pub struct StubFileWritePort; impl FileWritePort for StubFileWritePort { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _pre_computed_hash: Option, ) -> Result { Ok(File::default()) } @@ -185,13 +185,11 @@ impl FileWritePort for StubFileWritePort { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, _file_id: &str, - _temp_path: &Path, + _blob_hash: &str, _size: u64, - _content_type: Option, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result<(String, i64), DomainError> { Ok((String::new(), 0)) @@ -478,40 +476,7 @@ impl FileUploadUseCase for StubFileUploadUseCase { _name: String, _folder_id: Option, _content_type: String, - _temp_path: &Path, - _size: u64, - _pre_computed_hash: Option, - ) -> Result { - Ok(FileDto::default()) - } - - async fn upload_file_from_path( - &self, - _name: String, - _folder_id: Option, - _content_type: String, - _file_path: &Path, - _pre_computed_hash: Option, - ) -> Result { - Ok(FileDto::default()) - } - - async fn create_file( - &self, - _parent_path: &str, - _filename: &str, - _content: &[u8], - _content_type: &str, - ) -> Result { - Ok(FileDto::default()) - } - - async fn update_file( - &self, - _path: &str, - _content: &[u8], - _content_type: &str, - _modified_at: Option, + _blob: StoredBlob, ) -> Result { Ok(FileDto::default()) } @@ -519,10 +484,8 @@ impl FileUploadUseCase for StubFileUploadUseCase { async fn update_file_streaming( &self, _path: &str, - _temp_path: &Path, - _size: u64, + _blob: StoredBlob, _content_type: &str, - _pre_computed_hash: Option, _modified_at: Option, ) -> Result { Ok(FileDto::default()) @@ -756,25 +719,11 @@ impl SearchUseCase for StubSearchUseCase { // DedupPort // --------------------------------------------------------------------------- -use crate::application::ports::dedup_ports::{ - BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto, -}; +use crate::application::ports::dedup_ports::{BlobMetadataDto, DedupPort, DedupStatsDto}; pub struct StubDedupPort; impl DedupPort for StubDedupPort { - async fn store_from_file( - &self, - _source_path: &Path, - _content_type: Option, - _pre_computed_hash: Option, - ) -> Result { - Err(DomainError::internal_error( - "DedupService", - "DedupService not initialized", - )) - } - async fn blob_exists(&self, _hash: &str) -> bool { false } diff --git a/src/common/temp.rs b/src/common/temp.rs deleted file mode 100644 index b83729df..00000000 --- a/src/common/temp.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Shared helper for creating upload spool temp files. -//! -//! Upload paths spool the request body to a temp file before deduplication. -//! By default `tempfile` uses the OS temp dir (`std::env::temp_dir()`, i.e. -//! `$TMPDIR` / `/tmp`), which in many container setups is **tmpfs (RAM)**. -//! Writing a multi-hundred-MB upload there fills page-cache that counts -//! against the cgroup memory limit and can OOMKill the process. Pointing the -//! spool at a real-disk directory (`OXICLOUD_UPLOAD_TMPDIR`) keeps the upload -//! footprint proportional to the streaming buffer, not the file size. - -use std::path::Path; -use tempfile::NamedTempFile; - -/// Create a [`NamedTempFile`], honoring an optional configured spool directory. -/// -/// When `dir` is `Some`, the temp file is created there (the directory is -/// created if missing); otherwise the OS default temp dir is used. -pub fn new_spool_temp_file(dir: Option<&Path>) -> std::io::Result { - match dir { - Some(d) => { - std::fs::create_dir_all(d)?; - NamedTempFile::new_in(d) - } - None => NamedTempFile::new(), - } -} diff --git a/src/domain/repositories/file_repository.rs b/src/domain/repositories/file_repository.rs index 4fba9aae..d9878f2e 100644 --- a/src/domain/repositories/file_repository.rs +++ b/src/domain/repositories/file_repository.rs @@ -71,15 +71,15 @@ pub trait FileWriteRepository: Send + Sync + 'static { content: Vec, ) -> Result; - /// Streaming upload — saves a file from a temp file already on disk. - async fn save_file_from_temp( + /// Registers a file row pointing at a blob already stored in the + /// content-addressable chunk store (one blob reference is consumed). + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result; /// Moves a file to another folder. diff --git a/src/infrastructure/repositories/pg/file_blob_write_repository.rs b/src/infrastructure/repositories/pg/file_blob_write_repository.rs index 185509bf..c83208fe 100644 --- a/src/infrastructure/repositories/pg/file_blob_write_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_write_repository.rs @@ -220,26 +220,30 @@ impl FileBlobWriteRepository { Ok((new_hash.to_string(), updated_at)) } - /// Like [`FileWritePort::save_file_from_temp`] but also returns whether the - /// blob was genuinely new (`true`) or a dedup hit (`false`). - /// Used by [`FileUploadService`] to pass `is_new_blob` to lifecycle hooks. - pub async fn save_file_from_temp_with_dedup( + /// Register a file row pointing at a blob already stored in the chunk + /// store (the upload-ingest layer streamed the content in). Consumes the + /// caller's blob reference: any failure releases it before returning. + async fn save_file_with_blob_impl( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, - ) -> Result<(File, bool), DomainError> { - let user_id = self.resolve_user_id(folder_id.as_deref()).await?; - - let dedup_result = self - .dedup - .store_from_file(temp_path, Some(content_type.clone()), pre_computed_hash) - .await?; - let is_new_blob = !dedup_result.was_deduplicated(); - let blob_hash = dedup_result.hash().to_string(); + ) -> Result { + let user_id = match self.resolve_user_id(folder_id.as_deref()).await { + Ok(user_id) => user_id, + Err(e) => { + if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await { + tracing::error!( + "Blob orphaned after owner resolution failure — hash: {}, err: {}", + &blob_hash[..12], + rollback_err + ); + } + return Err(e); + } + }; // Deadlock victims (40P01) retry before the compensation below runs — // a successful retry must keep the blob reference alive. The final @@ -259,7 +263,7 @@ impl FileBlobWriteRepository { .bind(&name) .bind(&folder_id) .bind(user_id) - .bind(&blob_hash) + .bind(blob_hash) .bind(size as i64) .bind(&content_type) .bind(category_order_for(&name, &content_type)) @@ -269,7 +273,7 @@ impl FileBlobWriteRepository { { Ok(row) => row, Err(e) => { - if let Err(rollback_err) = self.dedup.remove_reference(&blob_hash).await { + if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await { tracing::error!( "Blob orphaned after failed INSERT — hash: {}, err: {}", &blob_hash[..12], @@ -309,32 +313,23 @@ impl FileBlobWriteRepository { row.1, row.2, Some(user_id), - blob_hash, + blob_hash.to_string(), )?; - Ok((file, is_new_blob)) + Ok(file) } } impl FileWritePort for FileBlobWriteRepository { - async fn save_file_from_temp( + async fn save_file_with_blob( &self, name: String, folder_id: Option, content_type: String, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - pre_computed_hash: Option, ) -> Result { - self.save_file_from_temp_with_dedup( - name, - folder_id, - content_type, - temp_path, - size, - pre_computed_hash, - ) - .await - .map(|(file, _)| file) + self.save_file_with_blob_impl(name, folder_id, content_type, blob_hash, size) + .await } async fn move_file( @@ -532,24 +527,18 @@ impl FileWritePort for FileBlobWriteRepository { Ok(()) } - async fn update_file_content_from_temp( + async fn update_file_content_with_blob( &self, file_id: &str, - temp_path: &std::path::Path, + blob_hash: &str, size: u64, - content_type: Option, - pre_computed_hash: Option, modified_at: Option, ) -> Result<(String, i64), DomainError> { - // Streaming: pass pre-computed hash so dedup skips re-reading the file. - let dedup_result = self - .dedup - .store_from_file(temp_path, content_type, pre_computed_hash) - .await?; - let new_hash = dedup_result.hash().to_string(); - + // The content was already ingested into the chunk store by the + // upload-ingest layer; swap_blob_hash consumes its reference and + // releases it on failure. let swapped = self - .swap_blob_hash(file_id, &new_hash, size as i64, modified_at) + .swap_blob_hash(file_id, blob_hash, size as i64, modified_at) .await?; // The file now maps to a different blob — drop the read-side cache // entry so streaming downloads cannot serve the previous content diff --git a/src/infrastructure/services/chunked_upload_service.rs b/src/infrastructure/services/chunked_upload_service.rs index 39aaa7d8..ffc9b8e5 100644 --- a/src/infrastructure/services/chunked_upload_service.rs +++ b/src/infrastructure/services/chunked_upload_service.rs @@ -6,14 +6,14 @@ //! (updated atomically on each chunk) are stored alongside the chunk files. //! On boot the service scans `temp_base_dir` and recovers any active sessions. //! - Parallel chunk transfers (up to 6 concurrent) -//! - Automatic reassembly with hash-on-write (BLAKE3) +//! - Completion streams the ordered parts straight into the CDC blob store //! - Expiration cleanup (24 h) //! //! Protocol: //! 1. POST /api/uploads → Create upload session, get upload_id //! 2. PATCH /api/uploads/:id → Upload chunks (parallel OK) //! 3. HEAD /api/uploads/:id → Check progress -//! 4. POST /api/uploads/:id/complete → Finalize and assemble +//! 4. POST /api/uploads/:id/complete → Stream parts into the blob store use chrono::{DateTime, Utc}; use dashmap::DashMap; @@ -28,7 +28,8 @@ use tokio::io::AsyncWriteExt; use uuid::Uuid; use crate::application::ports::chunked_upload_ports::{ - ChunkUploadResponseDto, ChunkedUploadPort, CreateUploadResponseDto, UploadStatusResponseDto, + ChunkUploadResponseDto, ChunkedUploadPort, CompletedUploadParts, CreateUploadResponseDto, + UploadStatusResponseDto, }; use crate::domain::errors::{DomainError, ErrorKind}; @@ -603,7 +604,7 @@ impl ChunkedUploadService { /// /// Used by the streaming REST PUT path: the handler calls /// `prepare_chunk` → streams body to disk via - /// `interfaces::upload_spool::stream_body_to_path` → calls + /// `interfaces::upload_ingest::stream_body_to_path` → calls /// `commit_chunk` to finalise. This lets the body bypass the /// in-memory `Bytes` allocation entirely (peak heap ~one HTTP /// frame instead of "chunk size"). @@ -936,24 +937,23 @@ impl ChunkedUploadService { }) } - /// Assemble chunks into final file and return the path + pre-computed BLAKE3 hash. + /// Validate completion and return the chunk parts in assembly order. /// - /// **Hash-on-Write**: BLAKE3 is computed while copying chunks into the - /// assembled file, eliminating the second sequential read that dedup_service - /// would otherwise need. - /// - /// Returns `(assembled_file_path, filename, folder_id, content_type, total_size, blake3_hash)`. + /// No assembled file is written and nothing is hashed here — the caller + /// streams the parts straight into the CDC chunk store, which computes + /// BLAKE3 and dedup-checks each CDC chunk in that single read pass. The + /// part files stay on disk until `finalize_upload`, so a completion + /// that fails downstream (e.g. client checksum mismatch) is retryable. async fn complete_upload_inner( &self, upload_id: &str, user_id: &str, - ) -> Result<(PathBuf, String, Option, String, u64, String), String> { - // Verify ownership before assembly + ) -> Result { + // Verify ownership before completion self.verify_session_owner(upload_id, user_id)?; // Get session and validate completion. - // Clone the session data and drop the DashMap ref immediately - // so the shard is not held during the expensive assembly step. + // Clone the session data and drop the DashMap ref immediately. let session = { let entry = self .sessions @@ -971,118 +971,29 @@ impl ChunkedUploadService { entry.clone() }; - // Assemble file with hash-on-write. - // - // The entire loop is offloaded to spawn_blocking because BLAKE3 - // hashing is CPU-bound and would otherwise block a Tokio worker, - // starving all other connections. - // Synchronous I/O is used inside the blocking thread — it avoids - // the async reactor overhead and is actually faster for this - // sequential workload. - let assembled_path = session.temp_dir.join("assembled"); - let chunks_meta: Vec<(usize, PathBuf)> = session - .chunks + // Chunk indices are unique (upload_chunk rejects duplicates) but may + // have arrived out of order — sort to recover the assembly order. + let mut indices: Vec = session.chunks.iter().map(|c| c.index).collect(); + indices.sort_unstable(); + let chunk_paths: Vec = indices .iter() - .map(|c| { - ( - c.index, - session.temp_dir.join(format!("chunk_{:06}", c.index)), - ) - }) + .map(|index| session.temp_dir.join(format!("chunk_{:06}", index))) .collect(); - let total_size = session.total_size; - - let hash = tokio::task::spawn_blocking(move || -> Result { - use std::io::{BufWriter as StdBufWriter, Read, Write}; - - let raw_output = std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&assembled_path) - .map_err(|e| format!("Failed to create assembled file: {e}"))?; - - // Pre-allocate assembled file to reduce fragmentation - let _ = raw_output.set_len(total_size); - - // 512 KB I/O buffers — 8× fewer syscalls than 64 KB - let mut output = StdBufWriter::with_capacity(524_288, raw_output); - let mut hasher = blake3::Hasher::new(); - - // For files >10 MB, use multithreaded BLAKE3 hashing (all cores) - const RAYON_THRESHOLD: u64 = 10 * 1024 * 1024; - let use_rayon = total_size > RAYON_THRESHOLD; - - // Single 512 KB read buffer reused across all chunks (avoids N allocations) - let mut buf = vec![0u8; 524_288]; - for (index, chunk_path) in &chunks_meta { - let mut chunk_file = std::fs::File::open(chunk_path) - .map_err(|e| format!("Failed to open chunk {index}: {e}"))?; - loop { - let n = chunk_file - .read(&mut buf) - .map_err(|e| format!("Failed to read chunk {index}: {e}"))?; - if n == 0 { - break; - } - if use_rayon { - hasher.update_rayon(&buf[..n]); - } else { - hasher.update(&buf[..n]); - } - output.write_all(&buf[..n]).map_err(|e| { - format!("Failed to write chunk {index} to assembled file: {e}") - })?; - } - } - - output - .flush() - .map_err(|e| format!("Failed to flush assembled file: {e}"))?; - // ── Durability boundary ──────────────────────────────────── - // `flush` drains BufWriter's userspace buffer but leaves the - // bytes in the kernel page cache. Without `sync_all`, a - // power loss between this `complete_upload` returning 2xx - // and the OS writeback timer firing (~5 s default) loses - // the merged blob — and PG's metadata row references a hash - // that no longer exists on disk. Reclaim the BufWriter's - // inner File via `into_inner` so we can `sync_all` it; the - // BufWriter would otherwise drop without flushing on the - // inner handle. - let raw_output = output - .into_inner() - .map_err(|e| format!("into_inner on BufWriter failed: {e}"))?; - raw_output - .sync_all() - .map_err(|e| format!("Failed to fsync assembled file: {e}"))?; - - // Clean up chunk files (keep assembled) — already on a blocking thread - for (_index, chunk_path) in &chunks_meta { - let _ = std::fs::remove_file(chunk_path); - } - - Ok(hasher.finalize().to_hex().to_string()) - }) - .await - .map_err(|e| format!("Assembly task panicked: {e}"))??; - - let assembled_path = session.temp_dir.join("assembled"); tracing::info!( - "✅ Assembled chunked upload: {} ({} bytes from {} chunks)", + "✅ Chunked upload complete: {} ({} bytes in {} chunks, streamed to CAS)", session.filename, session.total_size, - session.chunks.len() + chunk_paths.len() ); - Ok(( - assembled_path, - session.filename.clone(), - session.folder_id.clone(), - session.content_type.clone(), - session.total_size, - hash, - )) + Ok(CompletedUploadParts { + chunk_paths, + filename: session.filename.clone(), + folder_id: session.folder_id.clone(), + content_type: session.content_type.clone(), + total_size: session.total_size, + }) } /// Finalize upload: remove session from RAM, then clean disk OUTSIDE lock. @@ -1187,7 +1098,7 @@ impl ChunkedUploadPort for ChunkedUploadService { &self, upload_id: &str, user_id: Uuid, - ) -> Result<(PathBuf, String, Option, String, u64, String), DomainError> { + ) -> Result { self.complete_upload_inner(upload_id, &user_id.to_string()) .await .map_err(|e| DomainError::new(ErrorKind::InternalError, "ChunkedUpload", e)) @@ -1458,18 +1369,21 @@ mod tests { assert_eq!(status.completed_chunks, 2); assert!(status.pending_chunks.is_empty()); - // 4. Complete (assemble) - let (path, filename, _folder, _ct, size, hash) = service + // 4. Complete — returns the ordered chunk parts, no assembled file + let parts = service .complete_upload_inner(&id, "test-user") .await .expect("complete"); - assert_eq!(filename, "test.txt"); - assert_eq!(size, 1024); - assert!(!hash.is_empty()); - assert!(path.exists()); + assert_eq!(parts.filename, "test.txt"); + assert_eq!(parts.total_size, 1024); + assert_eq!(parts.chunk_paths.len(), 2); - // 5. Verify assembled content - let content = fs::read(&path).await.expect("read assembled"); + // 5. Verify the parts concatenate to the original content in order + let mut content = Vec::new(); + for path in &parts.chunk_paths { + assert!(path.exists(), "chunk part must remain until finalize"); + content.extend_from_slice(&fs::read(path).await.expect("read part")); + } assert_eq!(&content[..512], &[b'A'; 512]); assert_eq!(&content[512..], &[b'B'; 512]); diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 2d06f6ec..867c0767 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -18,33 +18,41 @@ //! blobs in `storage.blobs`) are served transparently — when no manifest //! row exists for a hash, the service falls back to direct blob reads. //! -//! **Write-first strategy** (store_from_file): -//! 1. CDC-analyse the file (mmap → FastCDC boundaries + per-chunk BLAKE3). -//! 2. Batch-check which chunk hashes already exist in PG (dedup skip). -//! 3. Bump ref_count for existing chunks (no disk I/O). -//! 4. Read + write only *new* chunks to the blob backend (idempotent, -//! no per-chunk fsync). -//! 5. One batched fsync sweep makes the new chunks durable, then ONE -//! batched INSERT registers them — durability before visibility. -//! 6. Single manifest INSERT (~few ms total). -//! 7. PG connection is never held during disk I/O. +//! **Single-pass streaming ingest** (store_from_stream): +//! 1. FastCDC boundaries, per-chunk BLAKE3 and the whole-file BLAKE3 are +//! all computed WHILE the bytes arrive — no spool file, no mmap +//! re-read. Peak RAM stays bounded (current chunk + one small batch). +//! 2. Per batch of distinct chunks, ONE `UPDATE … RETURNING` bumps +//! ref_count on already-known chunks (pinning them against concurrent +//! reclaim for the rest of the upload) and atomically classifies the +//! rest as new — no check-then-bump TOCTOU window. +//! 3. Only *new* chunks are written to the blob backend (unsynced, +//! bounded concurrency). Bytes the store already knows never touch +//! disk — a full dedup hit performs zero content writes. +//! 4. At end of stream ONE batched fsync sweep makes the new chunks +//! durable, then ONE batched INSERT registers them — durability +//! before visibility. +//! 5. Single manifest INSERT (~few ms). An identical concurrent upload +//! is resolved via ON CONFLICT: the loser releases its chunk +//! references and turns into a dedup hit. +//! 6. PG connections are never held during disk I/O. //! //! Benefits: +//! - Each uploaded byte hits the disk at most ONCE (dedup hits: zero) //! - Sub-file dedup: edited files share unchanged chunks //! - ACID durability — crash-safe, zero orphaned index entries -//! - PG connections never blocked by disk I/O (write-first) //! - 60-80% storage reduction for versioned / edited files -//! - Faster uploads when chunks already exist use bytes::Bytes; use futures::stream::{self, StreamExt}; use futures::{Stream, TryStreamExt}; use sqlx::PgPool; +use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; -use tokio::fs; +use tokio_util::io::StreamReader; use crate::application::ports::blob_lifecycle::BlobLifecycleHook; use crate::application::ports::blob_storage_ports::BlobStorageBackend; @@ -57,19 +65,171 @@ use crate::domain::errors::{DomainError, ErrorKind}; // ── CDC Constants ──────────────────────────────────────────────────────────── /// Minimum CDC chunk size (64 KB). -const CDC_MIN_CHUNK: usize = 65_536; +pub const CDC_MIN_CHUNK: usize = 65_536; /// Average CDC chunk size (256 KB). -const CDC_AVG_CHUNK: usize = 262_144; +pub const CDC_AVG_CHUNK: usize = 262_144; /// Maximum CDC chunk size (1 MB). -const CDC_MAX_CHUNK: usize = 1_048_576; +pub const CDC_MAX_CHUNK: usize = 1_048_576; // ── CDC helper types ───────────────────────────────────────────────────────── -/// Metadata for a single CDC chunk (offset + length + BLAKE3 hash). -struct ChunkMeta { - hash: String, - offset: usize, - length: usize, +/// Everything a streaming chunk ingest learned about its byte stream. +/// +/// Produced by [`DedupService::ingest_chunks_from_stream`]. On success the +/// ingest session holds exactly ONE `storage.blobs.ref_count` reference per +/// *distinct* chunk hash; the caller must either attach those references to +/// a manifest or hand them back via `release_chunk_refs`. +struct ChunkIngestOutcome { + /// BLAKE3 of the complete byte stream (the future manifest key). + file_hash: String, + /// Total bytes consumed from the stream. + total_size: u64, + /// Per-occurrence chunk hashes, in file order (the manifest layout). + chunk_hashes: Vec, + /// Per-occurrence chunk sizes, in file order. + chunk_sizes: Vec, + /// How many distinct chunks were actually written to the backend. + newly_written: usize, +} + +impl ChunkIngestOutcome { + /// Distinct chunk hashes — the set this ingest holds one reference on each. + fn distinct_hashes(&self) -> Vec { + let mut seen = HashSet::new(); + self.chunk_hashes + .iter() + .filter(|h| seen.insert(h.as_str())) + .cloned() + .collect() + } +} + +/// Compensation guard for an in-flight ingest session. +/// +/// Tracks the two side effects a session accumulates before its chunks are +/// fully registered: ref_count pins taken on pre-existing chunks and freshly +/// written (still unregistered) chunk files. If the session future is dropped +/// mid-stream — a client disconnect aborts the whole handler future — the +/// guard spawns a rollback so pinned chunks don't leak references forever and +/// written files become GC-collectible rows instead of invisible orphans. +struct IngestGuard { + pool: Arc, + backend: Arc, + /// Pre-existing chunks whose ref_count this session bumped (distinct). + pinned: Vec, + /// Chunks written to the backend but not yet registered: (hash, size). + written: Vec<(String, i64)>, + armed: bool, +} + +impl IngestGuard { + fn new(pool: Arc, backend: Arc) -> Self { + Self { + pool, + backend, + pinned: Vec::new(), + written: Vec::new(), + armed: true, + } + } + + /// The session's chunks are fully registered — references now belong to + /// the caller, nothing to compensate. + fn disarm(mut self) { + self.armed = false; + } + + /// Deterministic rollback for handled errors (awaited inline, unlike the + /// spawned Drop path). + async fn rollback(mut self) { + self.armed = false; + let pinned = std::mem::take(&mut self.pinned); + let written = std::mem::take(&mut self.written); + Self::run_rollback(self.pool.clone(), self.backend.clone(), pinned, written).await; + } + + /// Release pins and surface written-but-unregistered chunk files to GC. + /// + /// Best-effort: every step logs instead of failing — the worst outcome of + /// a failed rollback is a bounded ref_count over-count (storage leak), + /// never data loss. + async fn run_rollback( + pool: Arc, + backend: Arc, + pinned: Vec, + written: Vec<(String, i64)>, + ) { + if !pinned.is_empty() + && let Err(e) = sqlx::query( + "UPDATE storage.blobs SET ref_count = GREATEST(ref_count - 1, 0) + WHERE hash = ANY($1)", + ) + .bind(&pinned) + .execute(pool.as_ref()) + .await + { + tracing::warn!( + "Ingest rollback: failed to release {} chunk pins: {e}", + pinned.len() + ); + } + + if written.is_empty() { + return; + } + // Durability first, then visibility at ref_count 0 so the existing GC + // sweep can reclaim the bytes — a backend file with no PG row would be + // invisible to it. ON CONFLICT DO NOTHING keeps a concurrent + // uploader's row (and its references) intact. + let hashes: Vec = written.iter().map(|(h, _)| h.clone()).collect(); + let sizes: Vec = written.iter().map(|(_, s)| *s).collect(); + if let Err(e) = backend.sync_blobs(&hashes).await { + tracing::warn!( + "Ingest rollback: sync of {} chunks failed: {e}", + hashes.len() + ); + } + if let Err(e) = sqlx::query( + "INSERT INTO storage.blobs (hash, size, ref_count) + SELECT h, s, 0 FROM UNNEST($1::text[], $2::bigint[]) AS t(h, s) + ON CONFLICT (hash) DO NOTHING", + ) + .bind(&hashes) + .bind(&sizes) + .execute(pool.as_ref()) + .await + { + tracing::warn!( + "Ingest rollback: failed to register {} orphan chunks for GC: {e}", + hashes.len() + ); + } + } +} + +impl Drop for IngestGuard { + fn drop(&mut self) { + if !self.armed || (self.pinned.is_empty() && self.written.is_empty()) { + return; + } + let pinned = std::mem::take(&mut self.pinned); + let written = std::mem::take(&mut self.written); + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + let pool = self.pool.clone(); + let backend = self.backend.clone(); + handle.spawn(async move { + Self::run_rollback(pool, backend, pinned, written).await; + }); + } + Err(_) => tracing::warn!( + "Ingest guard dropped outside a runtime: {} pins / {} written chunks \ + stay leaked until the next GC sweep", + pinned.len(), + written.len() + ), + } + } } /// Content-Addressable Storage Service with CDC (PostgreSQL-backed) @@ -82,7 +242,7 @@ pub struct DedupService { /// Pluggable blob storage backend (local FS, S3, …). backend: Arc, /// PostgreSQL connection pool (dedup index in `storage.blobs`) — primary, - /// used by request-path operations (store_from_file, etc.). + /// used by request-path operations (store_from_stream, etc.). pool: Arc, /// Isolated maintenance pool for long-running operations /// (verify_integrity, garbage_collect) that must never starve the primary. @@ -192,86 +352,12 @@ impl DedupService { .unwrap_or_else(|| PathBuf::from(format!("remote://{}", hash))) } - // ── CDC analysis ─────────────────────────────────────────── - - /// Single-pass CDC: compute whole-file BLAKE3 hash + chunk boundaries + per-chunk hashes. - /// - /// Memory-maps the file and runs FastCDC boundary detection - /// concurrently with BLAKE3 hashing — all in one pass. - async fn cdc_hash_and_chunk_file(path: &Path) -> std::io::Result<(String, Vec)> { - let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&path)?; - let file_size = file.metadata()?.len(); - - if file_size == 0 { - return Ok((blake3::hash(b"").to_hex().to_string(), vec![])); - } - - // SAFETY: file is opened read-only; no concurrent writers expected - // (source is a temp upload file owned exclusively by this request). - let mmap = unsafe { memmap2::Mmap::map(&file)? }; - let chunker = - fastcdc::v2020::FastCDC::new(&mmap, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK); - - let mut file_hasher = blake3::Hasher::new(); - let mut chunks = Vec::new(); - - for chunk in chunker { - let data = &mmap[chunk.offset..chunk.offset + chunk.length]; - file_hasher.update(data); - chunks.push(ChunkMeta { - hash: blake3::hash(data).to_hex().to_string(), - offset: chunk.offset, - length: chunk.length, - }); - } - - Ok((file_hasher.finalize().to_hex().to_string(), chunks)) - }) - .await - .expect("cdc_hash_and_chunk_file: spawn_blocking panicked") - } - - /// CDC analysis without file-hash computation (when hash is pre-computed). - async fn cdc_chunk_file(path: &Path) -> std::io::Result> { - let path = path.to_path_buf(); - tokio::task::spawn_blocking(move || { - let file = std::fs::File::open(&path)?; - let file_size = file.metadata()?.len(); - - if file_size == 0 { - return Ok(vec![]); - } - - let mmap = unsafe { memmap2::Mmap::map(&file)? }; - let chunker = - fastcdc::v2020::FastCDC::new(&mmap, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK); - - let chunks: Vec = chunker - .map(|chunk| { - let data = &mmap[chunk.offset..chunk.offset + chunk.length]; - ChunkMeta { - hash: blake3::hash(data).to_hex().to_string(), - offset: chunk.offset, - length: chunk.length, - } - }) - .collect(); - - Ok(chunks) - }) - .await - .expect("cdc_chunk_file: spawn_blocking panicked") - } - // ── Hash helpers ───────────────────────────────────────────── /// Calculate BLAKE3 hash of a file (~5× faster than SHA-256). /// - /// Uses memory-mapped I/O with rayon parallelism. Kept for callers - /// that only need the hash (e.g. upload handlers pre-computing the hash - /// before calling `store_from_file`). + /// Uses memory-mapped I/O with rayon parallelism. Used by + /// `verify_integrity` to re-hash local blob files. pub async fn hash_file(path: &Path) -> std::io::Result { let path = path.to_path_buf(); tokio::task::spawn_blocking(move || { @@ -283,329 +369,541 @@ impl DedupService { .expect("hash_file: spawn_blocking task panicked") } - // ── Core store operations ──────────────────────────────────── - - /// Store content with CDC deduplication (from file). - /// - /// **Fast path**: if `pre_computed_hash` is `Some`, the manifest / - /// legacy-blob index is checked *before* running CDC — returning - /// instantly on a full-file dedup hit. - /// - /// **New-file path**: CDC-analyses the file (single mmap pass), - /// stores unique chunks via the blob backend, then inserts the - /// manifest in PostgreSQL. - pub async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result { - // ── Fast path: pre-computed hash → check before CDC ────── - if let Some(ref hash) = pre_computed_hash - && let Some(result) = self.try_dedup_hit(hash, source_path).await? - { - return Ok(result); - } - - // ── CDC analysis ───────────────────────────────────────── - let (file_hash, chunks) = if let Some(hash) = pre_computed_hash { - let chunks = Self::cdc_chunk_file(source_path) - .await - .map_err(DomainError::from)?; - (hash, chunks) - } else { - let (hash, chunks) = Self::cdc_hash_and_chunk_file(source_path) - .await - .map_err(DomainError::from)?; - // Check dedup with newly computed hash - if let Some(result) = self.try_dedup_hit(&hash, source_path).await? { - return Ok(result); - } - (hash, chunks) - }; - - let file_size = fs::metadata(source_path) - .await - .map_err(DomainError::from)? - .len(); - - // ── Store chunks (write-first — no PG connection held) ─── - let (chunk_hashes, chunk_sizes) = self.store_chunks(source_path, &chunks).await?; - - // ── Insert manifest ────────────────────────────────────── - sqlx::query( - "INSERT INTO storage.chunk_manifests - (file_hash, chunk_hashes, chunk_sizes, total_size, chunk_count, content_type, ref_count) - VALUES ($1, $2, $3, $4, $5, $6, 1)", - ) - .bind(&file_hash) - .bind(&chunk_hashes) - .bind(chunk_sizes.iter().map(|s| *s as i64).collect::>()) - .bind(file_size as i64) - .bind(chunk_hashes.len() as i32) - .bind(&content_type) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to insert manifest: {}", e)) - })?; - - // ── Clean up source file ───────────────────────────────── - let _ = fs::remove_file(source_path).await; - - tracing::info!( - "NEW BLOB (CDC): {} ({} bytes, {} chunks)", - &file_hash[..12], - file_size, - chunk_hashes.len() - ); - - self.fire_blob_creation_hooks(&file_hash, content_type.as_deref()); - - Ok(DedupResultDto::NewBlob { - hash: file_hash, - size: file_size, - }) - } - - /// Check manifest or legacy blob for a dedup hit. - /// - /// Returns `Some(ExistingBlob)` if the exact file was already stored. - /// Bumps the appropriate ref_count and removes the source file. - async fn try_dedup_hit( - &self, - hash: &str, - source_path: &Path, - ) -> Result, DomainError> { - // ── CDC manifest hit ───────────────────────────────────── - let manifest = sqlx::query_as::<_, (i64,)>( - "SELECT total_size FROM storage.chunk_manifests WHERE file_hash = $1", - ) - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to check manifest: {}", e)) - })?; - - if let Some((total_size,)) = manifest { - sqlx::query( - "UPDATE storage.chunk_manifests SET ref_count = ref_count + 1 WHERE file_hash = $1", - ) - .bind(hash) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( - "Dedup", - format!("Failed to bump manifest ref_count: {}", e), - ) - })?; - - let _ = fs::remove_file(source_path).await; - - tracing::info!( - "DEDUP HIT (manifest): {} ({} bytes saved)", - &hash[..12], - total_size - ); - return Ok(Some(DedupResultDto::ExistingBlob { - hash: hash.to_owned(), - size: total_size as u64, - saved_bytes: total_size as u64, - })); - } - - // ── Legacy whole-file blob hit ─────────────────────────── - let legacy = sqlx::query_as::<_, (i64,)>("SELECT size FROM storage.blobs WHERE hash = $1") - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to check legacy blob: {}", e)) - })?; - - if let Some((size,)) = legacy { - sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = $1") - .bind(hash) - .execute(self.pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( - "Dedup", - format!("Failed to bump legacy ref_count: {}", e), - ) - })?; - - let _ = fs::remove_file(source_path).await; - tracing::info!( - "DEDUP HIT (legacy blob): {} ({} bytes saved)", - &hash[..12], - size - ); - return Ok(Some(DedupResultDto::ExistingBlob { - hash: hash.to_owned(), - size: size as u64, - saved_bytes: size as u64, - })); - } - - Ok(None) - } + // ── Core store operations (streaming CDC) ─────────────────── /// Maximum concurrent chunk uploads to the blob backend. const CHUNK_UPLOAD_CONCURRENCY: usize = 8; + /// Flush the pending distinct-chunk batch after this many chunks… + const FLUSH_MAX_CHUNKS: usize = 32; + /// …or after this many buffered bytes, whichever comes first. Together + /// with the ≤ 1 MiB chunk in flight this bounds peak RAM per upload to + /// ~9 MiB regardless of file size. + const FLUSH_MAX_BYTES: usize = 8 * 1024 * 1024; - /// Store CDC chunks via the blob backend + upsert in PG. + /// Store content with CDC deduplication, straight from a byte stream — + /// the single write path for every upload surface (REST multipart, + /// WebDAV PUT, NextCloud PUT, chunked-upload assembly, WOPI PutFile). /// - /// Phase 0: Batch-queries PG to discover which chunk hashes already - /// exist in `storage.blobs`. - /// Phase 1: Bumps `ref_count` for chunks that already exist (one - /// batched UPDATE, no disk I/O — the biggest saving for versioned - /// files where most chunks are unchanged). - /// Phase 2: Reads + writes only *new* chunks, with up to - /// [`CHUNK_UPLOAD_CONCURRENCY`] writes in flight and **no per-chunk - /// fsync**. - /// Phase 3: One batched `sync_blobs` sweep makes every new chunk - /// durable (no-op for remote backends, which are durable on PUT). - /// Phase 4: ONE batched INSERT registers the new chunks in PG. The - /// sweep runs first so a crash can never leave a `storage.blobs` row - /// pointing at bytes that were still in the page cache. + /// One pass over the incoming bytes: FastCDC boundary detection, + /// per-chunk BLAKE3, the whole-file BLAKE3, dedup lookups and blob + /// writes all happen while the stream is still arriving. There is no + /// spool file and no re-read — each uploaded byte touches the disk at + /// most once, and not at all when the store already has its chunk. /// - /// `ref_count` is incremented once per *distinct* chunk (one reference per - /// manifest), staying symmetric with `remove_manifest_reference` so a file - /// that repeats a chunk cannot over-count and leak the blob forever. - async fn store_chunks( + /// Identical-content races (two clients uploading the same file + /// concurrently) are resolved at the manifest INSERT via ON CONFLICT: + /// the loser releases its chunk references and returns `ExistingBlob`. + pub async fn store_from_stream( &self, - source_path: &Path, - chunks: &[ChunkMeta], - ) -> Result<(Vec, Vec), DomainError> { - let pool = &self.pool; - let backend = &self.backend; + source: S, + content_type: Option, + ) -> Result + where + S: Stream> + Send, + { + let outcome = self.ingest_chunks_from_stream(source).await?; + tracing::debug!( + "CDC stream ingested: {} ({} bytes, {} chunks, {} written)", + &outcome.file_hash[..12], + outcome.total_size, + outcome.chunk_hashes.len(), + outcome.newly_written, + ); + let distinct = outcome.distinct_hashes(); + self.attach_manifest( + &outcome.file_hash, + &outcome.chunk_hashes, + &outcome.chunk_sizes, + outcome.total_size, + content_type, + &distinct, + ) + .await + } - // ── Phase 0: de-duplicate chunk hashes, then batch-check existence ── - // A single file can legitimately repeat the same chunk many times - // (zero-filled regions in disk/VM images, repeated document structures, - // concatenated archives). `ref_count` is tracked per *distinct* chunk — - // one reference per manifest — to stay symmetric with - // `remove_manifest_reference`, which decrements via - // `WHERE hash = ANY(chunk_hashes)` (matching each row once). Counting - // per-occurrence here would over-increment and leak the blob forever. - // Keep the first occurrence of each hash so new chunks know where to - // read their bytes. - let mut seen = std::collections::HashSet::new(); - let unique_chunks: Vec<&ChunkMeta> = chunks - .iter() - .filter(|c| seen.insert(c.hash.as_str())) - .collect(); - let unique_hashes: Vec = unique_chunks.iter().map(|c| c.hash.clone()).collect(); + /// Attach a manifest to chunk references the caller already holds (one + /// per distinct chunk hash) — the shared accounting tail of both + /// [`store_from_stream`] and the delta-upload commit. + /// + /// On a lost insert race or an already-existing manifest, the existing + /// manifest's ref_count is bumped FIRST and only then are the held chunk + /// references released (`distinct_held`); the reverse order could leave + /// the caller's file row without any manifest reference behind it. + pub async fn attach_manifest( + &self, + file_hash: &str, + chunk_hashes: &[String], + chunk_sizes: &[u64], + total_size: u64, + content_type: Option, + distinct_held: &[String], + ) -> Result { + // A bounded retry covers the rare interleaving where the manifest + // that beat our INSERT is deleted again before our ref bump lands. + for _ in 0..3 { + let inserted = sqlx::query( + "INSERT INTO storage.chunk_manifests + (file_hash, chunk_hashes, chunk_sizes, total_size, chunk_count, content_type, ref_count) + VALUES ($1, $2, $3, $4, $5, $6, 1) + ON CONFLICT (file_hash) DO NOTHING", + ) + .bind(file_hash) + .bind(chunk_hashes) + .bind(chunk_sizes.iter().map(|s| *s as i64).collect::>()) + .bind(total_size as i64) + .bind(chunk_hashes.len() as i32) + .bind(&content_type) + .execute(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to insert manifest: {}", e)) + })? + .rows_affected(); - let existing_hashes: std::collections::HashSet = - sqlx::query_scalar::<_, String>("SELECT hash FROM storage.blobs WHERE hash = ANY($1)") - .bind(&unique_hashes) - .fetch_all(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error( - "Dedup", - format!("Failed to check existing chunks: {}", e), - ) - })? - .into_iter() - .collect(); + if inserted > 0 { + tracing::info!( + "NEW BLOB (CDC): {} ({} bytes, {} chunks)", + &file_hash[..12], + total_size, + chunk_hashes.len(), + ); + self.fire_blob_creation_hooks(file_hash, content_type.as_deref()); + return Ok(DedupResultDto::NewBlob { + hash: file_hash.to_string(), + size: total_size, + }); + } - // ── Phase 1: bump ref_count for every existing chunk in ONE query ── - // (was one UPDATE per occurrence — now a single batched round-trip). - let existing: Vec = unique_hashes - .iter() - .filter(|h| existing_hashes.contains(*h)) - .cloned() - .collect(); - if !existing.is_empty() { - sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = ANY($1)") - .bind(&existing) - .execute(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to bump ref_count: {}", e)) - })?; + // The manifest already exists — either this exact content was + // stored before or an identical concurrent upload just won the + // race. Bump ITS ref_count, then hand back the held references. + if let Some(existing_size) = self.bump_manifest_if_exists(file_hash).await? { + self.release_chunk_refs(self.pool.as_ref(), distinct_held) + .await; + tracing::info!( + "DEDUP HIT (manifest): {} ({} bytes saved)", + &file_hash[..12], + existing_size, + ); + return Ok(DedupResultDto::ExistingBlob { + hash: file_hash.to_string(), + size: existing_size as u64, + saved_bytes: existing_size as u64, + }); + } } - // ── Phase 2: upload each NEW chunk once, concurrently ────────────── - // Read each new chunk by positioned I/O immediately before its upload - // instead of materializing every chunk's *data* up front. Peak heap for - // file content stays bounded to ~CHUNK_UPLOAD_CONCURRENCY × CDC_MAX_CHUNK - // (≈ 8 MiB) — proportional to the chunk size, never the file size. - // - // Owned metadata (hash + offset + length, no file data) so the stream - // below does not borrow `chunks` across an `.await` (which would make - // this future non-`Send` and break the upload handlers). - let new_ops: Vec<(String, u64, usize)> = unique_chunks - .iter() - .filter(|c| !existing_hashes.contains(&c.hash)) - .map(|c| (c.hash.clone(), c.offset as u64, c.length)) - .collect(); - - let source = Arc::new(std::fs::File::open(source_path).map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to open source file: {}", e)) - })?); - - // Writes are *unsynced*: no per-chunk fsync. Durability comes from - // the single batched sweep below, BEFORE any PG row references the - // new chunks — so a crash can never leave storage.blobs claiming a - // chunk whose bytes didn't reach the platter. - let results: Vec> = stream::iter(new_ops) - .map(|(hash, offset, length)| { - let source = source.clone(); - let backend = backend.clone(); - async move { - // Positioned read of just this chunk (≤ CDC_MAX_CHUNK) off - // the async runtime, then upload. - let bytes = tokio::task::spawn_blocking(move || { - use std::os::unix::fs::FileExt; - let mut buf = vec![0u8; length]; - source.read_exact_at(&mut buf, offset)?; - Ok::, std::io::Error>(buf) - }) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Read task failed: {}", e)) - })? - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to read chunk: {}", e)) - })?; - - backend - .put_blob_from_bytes_unsynced(&hash, Bytes::from(bytes)) - .await?; - Ok((hash, length as i64)) - } - }) - .buffer_unordered(Self::CHUNK_UPLOAD_CONCURRENCY) - .collect() + self.release_chunk_refs(self.pool.as_ref(), distinct_held) .await; + Err(DomainError::internal_error( + "Dedup", + format!("Manifest insert/bump kept racing for {file_hash}"), + )) + } - let mut new_rows: Vec<(String, i64)> = Vec::with_capacity(results.len()); - for result in results { - new_rows.push(result?); + // ── Delta-upload primitives ────────────────────────────────── + // + // The delta protocol ("upload only what changed") lets a client claim + // chunks by hash instead of sending their bytes. Two invariants keep + // that from becoming a content oracle or a poisoning vector: + // + // 1. **Ownership**: without bytes, a caller may only claim chunks that + // are already reachable through their own (non-trashed) files, or + // unreferenced orphans (ref_count = 0 — i.e. "I just uploaded it"). + // Everything else must be uploaded; the store dedups it on write. + // 2. **Verification**: a declared file_hash is never trusted — the + // commit re-reads the proposed chunk sequence server-side and + // recomputes BLAKE3 before any manifest row exists. A forged hash + // would otherwise poison future whole-file dedup hits for OTHER + // users uploading the genuine content. + // + // The download direction reuses invariant 1: a chunk's bytes are only + // served to callers whose own files already reference it. + + /// The ordered chunk list composing `file_hash`, for the delta-download + /// manifest: `(chunks[(hash, size)], total_size)`. + /// + /// Legacy whole-file blobs (pre-CDC, not yet re-chunked) are presented + /// as a single-chunk manifest of themselves — the chunk download path + /// can serve them directly, so sync clients need no special case. + pub async fn manifest_chunk_list( + &self, + file_hash: &str, + ) -> Result, u64)>, DomainError> { + let manifest = sqlx::query_as::<_, (Vec, Vec, i64)>( + "SELECT chunk_hashes, chunk_sizes, total_size + FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(file_hash) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("Dedup", format!("Manifest lookup: {e}")))?; + + if let Some((hashes, sizes, total)) = manifest { + let chunks = hashes + .into_iter() + .zip(sizes.into_iter().map(|s| s as u64)) + .collect(); + return Ok(Some((chunks, total as u64))); + } + + // Legacy fallback: the blob is its own single chunk. + let legacy = sqlx::query_scalar::<_, i64>("SELECT size FROM storage.blobs WHERE hash = $1") + .bind(file_hash) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Legacy blob lookup: {e}")) + })?; + Ok(legacy.map(|size| (vec![(file_hash.to_string(), size as u64)], size as u64))) + } + + /// Sizes of the given chunk hashes from the dedup index, keyed by hash. + /// Hashes without a row are simply absent from the result. + pub async fn chunk_sizes( + &self, + hashes: &[String], + ) -> Result, DomainError> { + if hashes.is_empty() { + return Ok(std::collections::HashMap::new()); + } + sqlx::query_as::<_, (String, i64)>( + "SELECT hash, size FROM storage.blobs WHERE hash = ANY($1)", + ) + .bind(hashes) + .fetch_all(self.pool.as_ref()) + .await + .map(|rows| rows.into_iter().map(|(h, s)| (h, s as u64)).collect()) + .map_err(|e| DomainError::internal_error("Dedup", format!("chunk_sizes query: {e}"))) + } + + /// Stream one chunk's raw bytes from the backend. The caller is + /// responsible for entitlement (see [`claimable_chunks`]). + pub async fn chunk_stream( + &self, + hash: &str, + ) -> Result> + Send>>, DomainError> + { + self.backend.get_blob_stream(hash).await + } + + /// Of `hashes` (distinct), the subset `caller_id` may claim without + /// uploading bytes: chunks referenced by manifests of the caller's + /// non-trashed files, or directly referenced as (legacy) whole-file + /// blobs. Backed by the GIN index on `chunk_manifests.chunk_hashes`. + pub async fn claimable_chunks( + &self, + caller_id: uuid::Uuid, + hashes: &[String], + ) -> Result, DomainError> { + if hashes.is_empty() { + return Ok(HashSet::new()); + } + sqlx::query_scalar::<_, String>( + "SELECT c.h FROM UNNEST($1::text[]) AS c(h) + WHERE EXISTS ( + SELECT 1 + FROM storage.files f + JOIN storage.chunk_manifests m ON m.file_hash = f.blob_hash + WHERE f.user_id = $2 AND NOT f.is_trashed + AND m.chunk_hashes @> ARRAY[c.h] + ) + OR EXISTS ( + SELECT 1 FROM storage.files f2 + WHERE f2.user_id = $2 AND NOT f2.is_trashed + AND f2.blob_hash = c.h + )", + ) + .bind(hashes) + .bind(caller_id) + .fetch_all(self.pool.as_ref()) + .await + .map(|rows| rows.into_iter().collect()) + .map_err(|e| DomainError::internal_error("Dedup", format!("claimable_chunks query: {e}"))) + } + + /// Pin one reference on each of `hashes` (distinct) that the caller is + /// entitled to claim — owned chunks (see [`claimable_chunks`]) or + /// unreferenced orphans (`ref_count = 0`, the just-uploaded state). + /// One statement: entitlement check and bump are atomic per row, so a + /// concurrent last-reference delete can never be resurrected and a + /// non-entitled hash is simply not returned. + /// + /// Returns the set actually pinned; the caller compares against its + /// input and reports the difference as `still_missing`. + pub async fn pin_claimable_chunks( + &self, + caller_id: uuid::Uuid, + hashes: &[String], + ) -> Result, DomainError> { + if hashes.is_empty() { + return Ok(HashSet::new()); + } + sqlx::query_scalar::<_, String>( + "UPDATE storage.blobs b + SET ref_count = ref_count + 1 + WHERE b.hash = ANY($1) + AND ( b.ref_count = 0 + OR EXISTS ( + SELECT 1 + FROM storage.files f + JOIN storage.chunk_manifests m ON m.file_hash = f.blob_hash + WHERE f.user_id = $2 AND NOT f.is_trashed + AND m.chunk_hashes @> ARRAY[b.hash::text] + ) + OR EXISTS ( + SELECT 1 FROM storage.files f2 + WHERE f2.user_id = $2 AND NOT f2.is_trashed + AND f2.blob_hash = b.hash + ) ) + RETURNING b.hash", + ) + .bind(hashes) + .bind(caller_id) + .fetch_all(self.pool.as_ref()) + .await + .map(|rows| rows.into_iter().collect()) + .map_err(|e| { + DomainError::internal_error("Dedup", format!("pin_claimable_chunks query: {e}")) + }) + } + + /// Release one reference per distinct hash — the public counterpart of + /// [`pin_claimable_chunks`] for aborted commits. Best-effort. + pub async fn release_pinned_chunks(&self, hashes: &[String]) { + self.release_chunk_refs(self.pool.as_ref(), hashes).await; + } + + /// Store client-provided loose chunks (delta upload, step 2). + /// + /// Each element of `frames` is one chunk's raw bytes (the wire framing + /// is the interface layer's concern). The hash is ALWAYS computed + /// server-side — a declared hash is never trusted for content + /// addressing. Chunks are written unsynced, made durable with one + /// batched sweep, then registered at `ref_count = 0`: unreferenced + /// orphans that either get pinned by a following commit or swept by + /// the periodic GC if the client never returns. `ON CONFLICT DO + /// NOTHING` keeps existing rows' reference counts untouched. + /// + /// Returns `(hash, size)` per frame, in input order. + pub async fn store_loose_chunks(&self, frames: S) -> Result, DomainError> + where + S: Stream> + Send, + { + futures::pin_mut!(frames); + + let mut received: Vec<(String, u64)> = Vec::new(); + let mut new_rows: Vec<(String, i64)> = Vec::new(); + let mut seen: HashSet = HashSet::new(); + + while let Some(frame) = frames.next().await { + let data = frame?; + if data.len() > CDC_MAX_CHUNK { + return Err(DomainError::validation_error(format!( + "Chunk frame of {} bytes exceeds the {CDC_MAX_CHUNK}-byte maximum", + data.len() + ))); + } + let hash = blake3::hash(&data).to_hex().to_string(); + received.push((hash.clone(), data.len() as u64)); + if seen.insert(hash.clone()) { + let len = data.len() as i64; + self.backend + .put_blob_from_bytes_unsynced(&hash, data) + .await?; + new_rows.push((hash, len)); + } } if !new_rows.is_empty() { - // ── Phase 3: durability barrier — one batched fsync sweep ────── - // (was 2 fsyncs per chunk: ~8 200 for a 1 GB upload; now one - // parallel sweep over the new files + ≤256 prefix dirs). - // Remote backends are durable on PUT — sync_blobs is a no-op. - let new_hashes: Vec = new_rows.iter().map(|(h, _)| h.clone()).collect(); - backend.sync_blobs(&new_hashes).await?; - - // ── Phase 4: register all new chunks in ONE batched INSERT ───── - // (was one round-trip per chunk). `new_rows` is built from - // `unique_chunks`, so no hash repeats within the batch — safe for - // ON CONFLICT DO UPDATE, which covers a concurrent uploader - // inserting the same brand-new chunk between the existence check - // in Phase 0 and this INSERT. - let new_sizes: Vec = new_rows.iter().map(|(_, s)| *s).collect(); + // Durability before visibility — same invariant as the ingest + // engine: no PG row may ever point at unsynced bytes. + let hashes: Vec = new_rows.iter().map(|(h, _)| h.clone()).collect(); + let sizes: Vec = new_rows.iter().map(|(_, s)| *s).collect(); + self.backend.sync_blobs(&hashes).await?; sqlx::query( + "INSERT INTO storage.blobs (hash, size, ref_count) + SELECT h, s, 0 FROM UNNEST($1::text[], $2::bigint[]) AS t(h, s) + ON CONFLICT (hash) DO NOTHING", + ) + .bind(&hashes) + .bind(&sizes) + .execute(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to register chunks: {e}")) + })?; + } + + Ok(received) + } + + /// Verification read for the delta commit: stream the proposed chunk + /// sequence from the backend, recompute the whole-file BLAKE3 and + /// capture the first bytes for MIME sniffing. The caller must hold a + /// pin on every chunk (so a concurrent GC cannot pull bytes out from + /// under the read). Also validates each chunk's actual size against + /// the declared one — the manifest's Range arithmetic depends on it. + pub async fn hash_chunk_sequence( + &self, + chunks: &[(String, u64)], + sniff_len: usize, + ) -> Result<(String, Vec), DomainError> { + let mut hasher = blake3::Hasher::new(); + let mut head: Vec = Vec::with_capacity(sniff_len.min(16 * 1024)); + + for (hash, declared_size) in chunks { + let mut stream = self.backend.get_blob_stream(hash).await?; + let mut actual: u64 = 0; + while let Some(part) = stream.next().await { + let part = part.map_err(|e| { + DomainError::internal_error( + "Dedup", + format!("Verification read of chunk {hash}: {e}"), + ) + })?; + actual += part.len() as u64; + hasher.update(&part); + if head.len() < sniff_len { + let take = (sniff_len - head.len()).min(part.len()); + head.extend_from_slice(&part[..take]); + } + } + if actual != *declared_size { + return Err(DomainError::validation_error(format!( + "Chunk {hash} is {actual} bytes, manifest declares {declared_size}" + ))); + } + } + + Ok((hasher.finalize().to_hex().to_string(), head)) + } + + /// Bump a manifest's ref_count if it exists; returns its total_size. + /// Single statement — no window between the existence check and the bump. + async fn bump_manifest_if_exists(&self, file_hash: &str) -> Result, DomainError> { + sqlx::query_scalar::<_, i64>( + "UPDATE storage.chunk_manifests SET ref_count = ref_count + 1 + WHERE file_hash = $1 + RETURNING total_size", + ) + .bind(file_hash) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to bump manifest ref_count: {e}")) + }) + } + + /// Stream → chunk store, WITHOUT creating a manifest. + /// + /// Splits the stream with FastCDC while computing per-chunk and + /// whole-stream BLAKE3 hashes, then settles each batch of distinct + /// chunks against PG: + /// + /// 1. ONE `UPDATE … RETURNING` per batch pins every already-known chunk + /// (`ref_count + 1` — protecting it from a concurrent last-reference + /// delete for the rest of the upload) and atomically classifies the + /// remaining hashes as new. No check-then-bump TOCTOU window. + /// 2. New chunks are written to the backend unsynced with bounded + /// concurrency; chunks the store already has are dropped from RAM + /// without any disk I/O. + /// 3. At end of stream, ONE `sync_blobs` sweep makes the new chunks + /// durable, then ONE batched INSERT registers them (`ON CONFLICT` + /// bumps instead — a concurrent identical upload may have registered + /// the same brand-new chunk first). Durability before visibility. + /// + /// `ref_count` is taken once per *distinct* chunk — symmetric with + /// `remove_manifest_reference`, which decrements via + /// `WHERE hash = ANY(chunk_hashes)` (each row once). A repeated chunk + /// (zero-filled regions, concatenated archives) must not over-count or + /// the blob leaks forever. + /// + /// If the returned references are not attached to a manifest, the caller + /// must hand them back via `release_chunk_refs`. If this future is + /// dropped mid-stream (client disconnect), the internal guard rolls the + /// session back in a spawned task. + async fn ingest_chunks_from_stream( + &self, + source: S, + ) -> Result + where + S: Stream> + Send, + { + let mut guard = IngestGuard::new(self.pool.clone(), self.backend.clone()); + + let reader = StreamReader::new(Box::pin(source)); + let mut chunker = fastcdc::v2020::AsyncStreamCDC::new( + reader, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ); + let chunk_stream = chunker.as_stream(); + futures::pin_mut!(chunk_stream); + + let mut file_hasher = blake3::Hasher::new(); + let mut total_size: u64 = 0; + let mut chunk_hashes: Vec = Vec::new(); + let mut chunk_sizes: Vec = Vec::new(); + let mut session_seen: HashSet = HashSet::new(); + let mut pending: Vec<(String, Bytes)> = Vec::new(); + let mut pending_bytes: usize = 0; + + while let Some(item) = chunk_stream.next().await { + let chunk = match item { + Ok(chunk) => chunk, + Err(e) => { + guard.rollback().await; + return Err(DomainError::internal_error( + "Dedup", + format!("Upload stream failed: {e}"), + )); + } + }; + + let data = chunk.data; + total_size += data.len() as u64; + // Per-chunk hashing is ≤ 1 MiB of BLAKE3 (< 1 ms) — cheaper than + // a spawn_blocking round-trip per chunk. + file_hasher.update(&data); + let hash = blake3::hash(&data).to_hex().to_string(); + chunk_sizes.push(data.len() as u64); + chunk_hashes.push(hash.clone()); + + if session_seen.insert(hash.clone()) { + pending_bytes += data.len(); + pending.push((hash, Bytes::from(data))); + if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES + { + if let Err(e) = self.flush_pending(&mut guard, &mut pending).await { + guard.rollback().await; + return Err(e); + } + pending_bytes = 0; + } + } + } + + if let Err(e) = self.flush_pending(&mut guard, &mut pending).await { + guard.rollback().await; + return Err(e); + } + + // ── Durability before visibility for the new chunks ────── + // One batched fsync sweep (no-op for remote backends, durable on + // PUT), then one batched INSERT. A crash before the INSERT leaves + // only unreferenced files; never a row pointing at unsynced bytes. + if !guard.written.is_empty() { + let new_hashes: Vec = guard.written.iter().map(|(h, _)| h.clone()).collect(); + let new_sizes: Vec = guard.written.iter().map(|(_, s)| *s).collect(); + + if let Err(e) = self.backend.sync_blobs(&new_hashes).await { + guard.rollback().await; + return Err(e); + } + + let registered = sqlx::query( "INSERT INTO storage.blobs (hash, size, ref_count) SELECT h, s, 1 FROM UNNEST($1::text[], $2::bigint[]) AS t(h, s) ON CONFLICT (hash) DO UPDATE @@ -613,19 +911,102 @@ impl DedupService { ) .bind(&new_hashes) .bind(&new_sizes) - .execute(pool.as_ref()) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to upsert chunks: {}", e)) - })?; + .execute(self.pool.as_ref()) + .await; + + if let Err(e) = registered { + guard.rollback().await; + return Err(DomainError::internal_error( + "Dedup", + format!("Failed to register chunks: {e}"), + )); + } } - // chunk_hashes/chunk_sizes keep the full per-occurrence CDC sequence — - // the manifest needs every chunk, in order, to reassemble the file. - let chunk_hashes: Vec = chunks.iter().map(|c| c.hash.clone()).collect(); - let chunk_sizes: Vec = chunks.iter().map(|c| c.length as u64).collect(); + let newly_written = guard.written.len(); + guard.disarm(); - Ok((chunk_hashes, chunk_sizes)) + Ok(ChunkIngestOutcome { + file_hash: file_hasher.finalize().to_hex().to_string(), + total_size, + chunk_hashes, + chunk_sizes, + newly_written, + }) + } + + /// Settle one batch of distinct in-RAM chunks against PG + the backend. + /// + /// Successfully pinned hashes and written chunks are recorded on the + /// guard as they happen, so a failure mid-batch leaves nothing + /// untracked for rollback. + async fn flush_pending( + &self, + guard: &mut IngestGuard, + pending: &mut Vec<(String, Bytes)>, + ) -> Result<(), DomainError> { + if pending.is_empty() { + return Ok(()); + } + let batch = std::mem::take(pending); + let hashes: Vec = batch.iter().map(|(h, _)| h.clone()).collect(); + + // Pin-or-classify in one statement: rows that exist take this + // session's reference NOW; hashes not returned don't exist and are + // ours to write. + let pinned: HashSet = sqlx::query_scalar::<_, String>( + "UPDATE storage.blobs SET ref_count = ref_count + 1 + WHERE hash = ANY($1) + RETURNING hash", + ) + .bind(&hashes) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("Failed to pin existing chunks: {e}")) + })? + .into_iter() + .collect(); + + let mut to_write: Vec<(String, Bytes)> = Vec::with_capacity(batch.len()); + for (hash, data) in batch { + if pinned.contains(&hash) { + guard.pinned.push(hash); + } else { + to_write.push((hash, data)); + } + } + if to_write.is_empty() { + return Ok(()); + } + + // Unsynced writes — durability comes from the single end-of-stream + // sweep, before any PG row references these chunks. + let backend = self.backend.clone(); + let results: Vec> = stream::iter(to_write) + .map(|(hash, data)| { + let backend = backend.clone(); + async move { + let len = data.len() as i64; + backend.put_blob_from_bytes_unsynced(&hash, data).await?; + Ok((hash, len)) + } + }) + .buffer_unordered(Self::CHUNK_UPLOAD_CONCURRENCY) + .collect() + .await; + + let mut first_err: Option = None; + for result in results { + match result { + Ok(row) => guard.written.push(row), + Err(e) => first_err = first_err.or(Some(e)), + } + } + match first_err { + Some(e) => Err(e), + None => Ok(()), + } } // ── Reference counting ─────────────────────────────────────── @@ -879,7 +1260,7 @@ impl DedupService { DomainError::internal_error("Dedup", format!("Failed to begin transaction: {}", e)) })?; - // Lock the row exclusively — prevents concurrent store_from_file from + // Lock the row exclusively — prevents a concurrent ingest from // incrementing ref_count while we might be deleting let row = sqlx::query_as::<_, (i32, i64)>( "SELECT ref_count, size FROM storage.blobs WHERE hash = $1 FOR UPDATE", @@ -917,7 +1298,7 @@ impl DedupService { })?; // Delete blob from backend AFTER committing PG — the row is gone, - // so no concurrent store_from_file can resurrect a reference. + // so no concurrent ingest can resurrect a reference. if let Err(e) = self.backend.delete_blob(hash).await { tracing::warn!("Failed to delete blob file {}: {}", hash, e); } @@ -1532,15 +1913,16 @@ impl DedupService { // report `legacy re-chunk: nothing to do`). // // Per-hash algorithm: - // 1. Spool the blob to a temp file via the normal read path (this - // decrypts it when encryption is on), verifying BLAKE3 == hash. - // 2. CDC-chunk the spool + store chunks (`store_chunks` bumps each - // distinct chunk once — the manifest's reference). - // 3. One short accounting TX with the blob row locked: + // 1. Stream the blob through the normal read path (this decrypts it + // when encryption is on) straight into the chunk-ingest engine — + // no spool file — verifying BLAKE3 == hash before keeping the + // chunks (each distinct chunk bumped once — the manifest's + // reference). + // 2. One short accounting TX with the blob row locked: // manifest INSERT with ref_count = N (current file rows referencing // the hash), blob ref_count -= N (those references now live on the // manifest), DELETE the blob row only if it hits exactly 0. - // 4. Physically delete the whole-file blob only when its row was + // 3. Physically delete the whole-file blob only when its row was // removed. Single-chunk files (chunk hash == file hash) keep the // physical blob — it IS the chunk; only the bookkeeping moves. // @@ -1548,7 +1930,7 @@ impl DedupService { // and the legacy dedup-hit path. A racing identical upload can land one // legacy reference after our commit; the blob row then survives (> 0) // and that file stays readable through the legacy fallback — a bounded - // space leak, never data loss. A crash between step 2 and 3 leaks one + // space leak, never data loss. A crash between step 1 and 2 leaks one // +1 on that file's chunk refs (re-run re-bumps); also a bounded leak, // never data loss. @@ -1688,22 +2070,12 @@ impl DedupService { hash: &str, content_type: Option, ) -> Result { - // ── 1. Spool + verify (decrypts via the normal read path) ── - // The spooled, hash-verified plaintext is the source of truth for - // sizes — `storage.blobs.size` is legacy metadata we don't trust + // ── 1. Stream + verify (decrypts via the normal read path) ── + // The chunk store is fed directly from the blob read stream — no + // spool file. Sizes come from the CDC pass over the hash-verified + // plaintext; `storage.blobs.size` is legacy metadata we don't trust // for the manifest's Range arithmetic. - // - // The path carries a per-attempt UUID: two processes sharing a temp - // dir and racing on the same hash must never truncate or delete each - // other's in-flight spool. - let spool = std::env::temp_dir().join(format!( - "oxicloud-rechunk-{}-{}.tmp", - &hash[..hash.len().min(16)], - uuid::Uuid::new_v4() - )); - let result = self.spool_and_chunk(hash, &spool).await; - let _ = fs::remove_file(&spool).await; - let (chunk_hashes, chunk_sizes) = result?; + let (chunk_hashes, chunk_sizes) = self.ingest_legacy_blob(hash).await?; let total_size: u64 = chunk_sizes.iter().sum(); // ── 2. Accounting TX: move the file references onto the manifest ── @@ -1756,10 +2128,11 @@ impl DedupService { if inserted == 0 { // A manifest appeared concurrently — only possible if the same - // content was re-uploaded and fully stored during our spool. + // content was re-uploaded and fully stored while we streamed. // Their bookkeeping is already correct; drop ours. tx.rollback().await.ok(); - self.release_chunk_refs(&chunk_hashes).await; + self.release_chunk_refs(self.maintenance_pool.as_ref(), &chunk_hashes) + .await; return Ok(0); } @@ -1828,63 +2201,36 @@ impl DedupService { Ok(freed) } - /// Spool a legacy blob to `spool`, verify its BLAKE3 matches `hash`, - /// CDC-chunk it and store the chunks. Returns (chunk_hashes, chunk_sizes). - async fn spool_and_chunk( - &self, - hash: &str, - spool: &Path, - ) -> Result<(Vec, Vec), DomainError> { - use tokio::io::AsyncWriteExt; - - let mut stream = self.read_blob_stream(hash).await?; - let file = fs::File::create(spool) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk spool: {e}")))?; - let mut writer = tokio::io::BufWriter::with_capacity(512 * 1024, file); - let mut hasher = blake3::Hasher::new(); - - while let Some(chunk) = stream.next().await { - let chunk = chunk - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk read: {e}")))?; - hasher.update(&chunk); - writer - .write_all(&chunk) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk write: {e}")))?; - } - writer - .flush() - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk flush: {e}")))?; - - let actual = hasher.finalize().to_hex().to_string(); - if actual != hash { + /// Re-chunk one legacy whole-file blob straight from the backend read + /// stream (no spool file), verifying that the streamed content still + /// matches its recorded BLAKE3 before the chunks are kept. + /// + /// On mismatch the freshly taken chunk references are released — the + /// written chunk bytes become unreferenced rows the GC sweeps — and an + /// error is returned; the legacy blob itself stays untouched. + async fn ingest_legacy_blob(&self, hash: &str) -> Result<(Vec, Vec), DomainError> { + let stream = self.read_blob_stream(hash).await?; + let outcome = self.ingest_chunks_from_stream(stream).await?; + if outcome.file_hash != hash { + let distinct = outcome.distinct_hashes(); + self.release_chunk_refs(self.maintenance_pool.as_ref(), &distinct) + .await; return Err(DomainError::internal_error( "Dedup", - format!("Blob content does not match its hash (expected {hash}, got {actual})"), + format!( + "Blob content does not match its hash (expected {hash}, got {})", + outcome.file_hash + ), )); } - - // Empty blobs can't be mmap'd by the CDC analyser; they become an - // empty manifest (the chunked read path streams zero chunks). - let spooled_len = fs::metadata(spool) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk stat: {e}")))? - .len(); - if spooled_len == 0 { - return Ok((Vec::new(), Vec::new())); - } - - let chunks = Self::cdc_chunk_file(spool) - .await - .map_err(DomainError::from)?; - self.store_chunks(spool, &chunks).await + Ok((outcome.chunk_hashes, outcome.chunk_sizes)) } - /// Best-effort compensation: drop the per-manifest chunk references - /// taken by `store_chunks` when the manifest insert was abandoned. - async fn release_chunk_refs(&self, chunk_hashes: &[String]) { + /// Best-effort compensation: drop one reference per *distinct* chunk + /// hash (clamped at 0). Used whenever an ingest session's references end + /// up not being attached to a manifest — dedup hit, lost insert race, or + /// content-verification failure. + async fn release_chunk_refs(&self, pool: &PgPool, chunk_hashes: &[String]) { if chunk_hashes.is_empty() { return; } @@ -1893,10 +2239,10 @@ impl DedupService { WHERE hash = ANY($1)", ) .bind(chunk_hashes) - .execute(self.maintenance_pool.as_ref()) + .execute(pool) .await { - tracing::warn!("Legacy re-chunk: failed to release chunk refs: {e}"); + tracing::warn!("Dedup: failed to release chunk refs: {e}"); } } } @@ -1915,16 +2261,6 @@ pub struct LegacyRechunkReport { // ─── Port implementation ───────────────────────────────────────────────────── impl DedupPort for DedupService { - async fn store_from_file( - &self, - source_path: &Path, - content_type: Option, - pre_computed_hash: Option, - ) -> Result { - self.store_from_file(source_path, content_type, pre_computed_hash) - .await - } - async fn blob_exists(&self, hash: &str) -> bool { self.blob_exists(hash).await } @@ -2002,20 +2338,95 @@ mod tests { file } + /// One chunk as seen by the streaming analyser. + struct TestChunk { + hash: String, + offset: usize, + length: usize, + } + + /// Run the exact same streaming chunker the ingest engine uses + /// (`AsyncStreamCDC` + the production CDC parameters) over an in-memory + /// buffer, feeding it in `frame`-sized pieces to exercise the refill + /// logic the same way HTTP body frames do. + /// + /// Returns the whole-stream BLAKE3 plus per-chunk metadata. + async fn stream_cdc(data: &[u8], frame: usize) -> (String, Vec) { + let frames: Vec> = data + .chunks(frame.max(1)) + .map(|c| Ok(Bytes::copy_from_slice(c))) + .collect(); + let reader = StreamReader::new(Box::pin(stream::iter(frames))); + let mut chunker = fastcdc::v2020::AsyncStreamCDC::new( + reader, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ); + let chunk_stream = chunker.as_stream(); + futures::pin_mut!(chunk_stream); + + let mut file_hasher = blake3::Hasher::new(); + let mut chunks = Vec::new(); + while let Some(item) = chunk_stream.next().await { + let chunk = item.expect("in-memory stream cannot fail"); + file_hasher.update(&chunk.data); + chunks.push(TestChunk { + hash: blake3::hash(&chunk.data).to_hex().to_string(), + offset: chunk.offset as usize, + length: chunk.length, + }); + } + (file_hasher.finalize().to_hex().to_string(), chunks) + } + + const TEST_FRAME: usize = 64 * 1024; // typical HTTP body frame size + + // ── Stream chunking ≡ slice chunking ───────────────────────── + // + // The whole dedup index hinges on this invariant: the boundaries (and + // therefore the chunk hashes) produced by the streaming chunker must be + // identical to FastCDC over the full in-memory slice, regardless of how + // the bytes were framed on the wire. Pre-streaming blobs were chunked + // via mmap + slice FastCDC — their chunks must keep deduplicating + // against newly streamed uploads. + + #[tokio::test] + async fn test_stream_chunking_matches_slice_chunking() { + let data: Vec = (0..4 * 1024 * 1024) + .map(|i| ((i as u64).wrapping_mul(6364136223846793005).wrapping_add(1)) as u8) + .collect(); + + let slice_chunks: Vec<(usize, usize)> = + fastcdc::v2020::FastCDC::new(&data, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK) + .map(|c| (c.offset, c.length)) + .collect(); + + for frame in [7usize, 4096, TEST_FRAME, data.len()] { + let (_, streamed) = stream_cdc(&data, frame).await; + assert_eq!( + streamed.len(), + slice_chunks.len(), + "chunk count must not depend on framing (frame={frame})" + ); + for (s, (offset, length)) in streamed.iter().zip(slice_chunks.iter()) { + assert_eq!((s.offset, s.length), (*offset, *length), "frame={frame}"); + let expected = blake3::hash(&data[*offset..*offset + *length]) + .to_hex() + .to_string(); + assert_eq!(s.hash, expected, "frame={frame}"); + } + } + } + // ── Determinism ────────────────────────────────────────────── #[tokio::test] async fn test_cdc_deterministic_same_content() { let data = vec![42u8; 512 * 1024]; // 512 KB of 0x2A - let f1 = write_temp_file(&data).await; - let f2 = write_temp_file(&data).await; - let (hash1, chunks1) = DedupService::cdc_hash_and_chunk_file(f1.path()) - .await - .unwrap(); - let (hash2, chunks2) = DedupService::cdc_hash_and_chunk_file(f2.path()) - .await - .unwrap(); + let (hash1, chunks1) = stream_cdc(&data, TEST_FRAME).await; + let (hash2, chunks2) = stream_cdc(&data, 4096).await; assert_eq!(hash1, hash2, "same content must produce same file hash"); assert_eq!( @@ -2030,16 +2441,13 @@ mod tests { } } - // ── Empty file ─────────────────────────────────────────────── + // ── Empty stream ───────────────────────────────────────────── #[tokio::test] - async fn test_cdc_empty_file() { - let f = write_temp_file(b"").await; - let (hash, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + async fn test_cdc_empty_stream() { + let (hash, chunks) = stream_cdc(b"", TEST_FRAME).await; - assert!(chunks.is_empty(), "empty file must produce zero chunks"); + assert!(chunks.is_empty(), "empty stream must produce zero chunks"); assert_eq!(hash, blake3::hash(b"").to_hex().to_string()); } @@ -2048,10 +2456,7 @@ mod tests { #[tokio::test] async fn test_cdc_small_file_single_chunk() { let data = b"Hello, OxiCloud CDC dedup!"; - let f = write_temp_file(data).await; - let (hash, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (hash, chunks) = stream_cdc(data, TEST_FRAME).await; assert_eq!(chunks.len(), 1, "tiny file must be a single chunk"); assert_eq!(chunks[0].offset, 0); @@ -2067,11 +2472,8 @@ mod tests { let data: Vec = (0..4 * 1024 * 1024) .map(|i| ((i as u64).wrapping_mul(6364136223846793005).wrapping_add(1)) as u8) .collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; assert!(chunks.len() > 1, "4 MB should produce multiple chunks"); @@ -2104,78 +2506,22 @@ mod tests { let data: Vec = (0..1024 * 1024).map(|i| (i % 251) as u8).collect(); let f = write_temp_file(&data).await; - let (cdc_hash, _) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (cdc_hash, _) = stream_cdc(&data, TEST_FRAME).await; let standalone_hash = DedupService::hash_file(f.path()).await.unwrap(); assert_eq!( cdc_hash, standalone_hash, - "CDC file hash must match standalone hash_file()" + "streamed file hash must match standalone hash_file()" ); } - // ── Chunk hashes are correct BLAKE3 of chunk data ──────────── - - #[tokio::test] - async fn test_cdc_chunk_hashes_are_correct() { - let data: Vec = (0..2 * 1024 * 1024) - .map(|i| ((i as u64).wrapping_mul(2862933555777941757).wrapping_add(3)) as u8) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - - for chunk in &chunks { - let chunk_data = &data[chunk.offset..chunk.offset + chunk.length]; - let expected_hash = blake3::hash(chunk_data).to_hex().to_string(); - assert_eq!( - chunk.hash, expected_hash, - "chunk at offset {} has wrong hash", - chunk.offset - ); - } - } - - // ── Reassembly matches original ────────────────────────────── - - #[tokio::test] - async fn test_cdc_reassembly_matches_original() { - let data: Vec = (0..3 * 1024 * 1024) - .map(|i| ((i as u64).wrapping_mul(1103515245).wrapping_add(12345)) as u8) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - - // Reassemble from chunks - let mut reassembled = Vec::with_capacity(data.len()); - for chunk in &chunks { - reassembled.extend_from_slice(&data[chunk.offset..chunk.offset + chunk.length]); - } - - assert_eq!( - reassembled.len(), - data.len(), - "reassembled length must match" - ); - assert_eq!(reassembled, data, "reassembled content must match original"); - } - - // ── Chunks cover entire file (no gaps, no overlaps) ────────── + // ── Reassembly: chunks are contiguous and cover the file ───── #[tokio::test] async fn test_cdc_chunks_are_contiguous() { let data: Vec = (0..2 * 1024 * 1024).map(|i| (i % 199) as u8).collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; let mut expected_offset = 0usize; for (i, chunk) in chunks.iter().enumerate() { @@ -2205,15 +2551,8 @@ mod tests { *b = b.wrapping_add(1); } - let f_base = write_temp_file(&base).await; - let f_mod = write_temp_file(&modified).await; - - let (hash_base, chunks_base) = DedupService::cdc_hash_and_chunk_file(f_base.path()) - .await - .unwrap(); - let (hash_mod, chunks_mod) = DedupService::cdc_hash_and_chunk_file(f_mod.path()) - .await - .unwrap(); + let (hash_base, chunks_base) = stream_cdc(&base, TEST_FRAME).await; + let (hash_mod, chunks_mod) = stream_cdc(&modified, TEST_FRAME).await; // File hashes must differ assert_ne!( @@ -2241,28 +2580,6 @@ mod tests { ); } - // ── cdc_chunk_file matches cdc_hash_and_chunk_file ─────────── - - #[tokio::test] - async fn test_cdc_chunk_file_matches_full() { - let data: Vec = (0..1024 * 1024) - .map(|i| (i as u8).wrapping_mul(7)) - .collect(); - let f = write_temp_file(&data).await; - - let (_, chunks_full) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); - let chunks_only = DedupService::cdc_chunk_file(f.path()).await.unwrap(); - - assert_eq!(chunks_full.len(), chunks_only.len()); - for (a, b) in chunks_full.iter().zip(chunks_only.iter()) { - assert_eq!(a.hash, b.hash); - assert_eq!(a.offset, b.offset); - assert_eq!(a.length, b.length); - } - } - // ── Large file produces expected chunk count ────────────────── #[tokio::test] @@ -2271,11 +2588,8 @@ mod tests { let data: Vec = (0..8 * 1024 * 1024) .map(|i| ((i as u64).wrapping_mul(2862933555777941757).wrapping_add(3)) as u8) .collect(); - let f = write_temp_file(&data).await; - let (_, chunks) = DedupService::cdc_hash_and_chunk_file(f.path()) - .await - .unwrap(); + let (_, chunks) = stream_cdc(&data, TEST_FRAME).await; // With 256KB avg, expect 20-60 chunks for 8MB assert!( @@ -2306,15 +2620,8 @@ mod tests { let mut with_prefix = prefix; with_prefix.extend_from_slice(&base); - let f_base = write_temp_file(&base).await; - let f_prefix = write_temp_file(&with_prefix).await; - - let (_, chunks_base) = DedupService::cdc_hash_and_chunk_file(f_base.path()) - .await - .unwrap(); - let (_, chunks_prefix) = DedupService::cdc_hash_and_chunk_file(f_prefix.path()) - .await - .unwrap(); + let (_, chunks_base) = stream_cdc(&base, TEST_FRAME).await; + let (_, chunks_prefix) = stream_cdc(&with_prefix, TEST_FRAME).await; let base_set: HashSet<&str> = chunks_base.iter().map(|c| c.hash.as_str()).collect(); let prefix_set: HashSet<&str> = chunks_prefix.iter().map(|c| c.hash.as_str()).collect(); @@ -2331,6 +2638,20 @@ mod tests { chunks_prefix.len() ); } + + // ── ChunkIngestOutcome helpers ─────────────────────────────── + + #[test] + fn test_distinct_hashes_deduplicates_preserving_order() { + let outcome = ChunkIngestOutcome { + file_hash: String::new(), + total_size: 0, + chunk_hashes: vec!["a".into(), "b".into(), "a".into(), "c".into(), "b".into()], + chunk_sizes: vec![1, 2, 1, 3, 2], + newly_written: 0, + }; + assert_eq!(outcome.distinct_hashes(), vec!["a", "b", "c"]); + } } // ───────────────────────────────────────────────────────────────────────────── @@ -2684,3 +3005,325 @@ mod rechunk_integration_tests { cleanup(&pool, &hash, &files).await; } } + +// ───────────────────────────────────────────────────────────────────────────── +// Integration tests for the delta-upload primitives — the entitlement and +// verification rules the chunk-negotiation protocol stands on. Same gating +// and DB conventions as the re-chunk suite above. +// ───────────────────────────────────────────────────────────────────────────── +#[cfg(integration_tests)] +#[allow(dead_code)] +mod delta_upload_integration_tests { + use super::*; + use crate::infrastructure::services::local_blob_backend::LocalBlobBackend; + use crate::integration_test_support::{ensure_clean_test_db, test_db_url}; + use sqlx::Row; + use sqlx::postgres::PgPoolOptions; + use tempfile::TempDir; + use uuid::Uuid; + + async fn test_pool() -> Arc { + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&test_db_url()) + .await + .expect("connect to test DB — run tests/common/spawn-db.sh first"); + ensure_clean_test_db(&pool).await; + Arc::new(pool) + } + + async fn seed_user(pool: &PgPool) -> Uuid { + sqlx::query("SELECT id FROM auth.users LIMIT 1") + .fetch_one(pool) + .await + .map(|r| r.get::("id")) + .expect("auth.users must be seeded (init-test-schema.sh)") + } + + async fn local_svc(pool: &Arc, dir: &TempDir) -> DedupService { + let backend = Arc::new(LocalBlobBackend::new(&dir.path().join("blobs"))); + backend.initialize().await.expect("init backend"); + DedupService::new(backend, pool.clone(), pool.clone()) + } + + /// Store `data` through the streaming path and give `user_id` a file + /// row referencing it — making its chunks claimable by that user. + async fn seed_owned_content( + svc: &DedupService, + pool: &PgPool, + user_id: Uuid, + data: &[u8], + label: &str, + ) -> (String, Vec, Uuid) { + let source = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::copy_from_slice(data))]); + let stored = svc + .store_from_stream(source, Some("application/octet-stream".into())) + .await + .expect("store"); + let file_hash = stored.hash().to_string(); + let chunks: Vec = sqlx::query_scalar( + "SELECT UNNEST(chunk_hashes) FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(&file_hash) + .fetch_all(pool) + .await + .expect("chunks"); + + let file_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.files (name, user_id, blob_hash, size) + VALUES ($1, $2, $3, $4) RETURNING id", + ) + .bind(format!( + "rust-test-delta-{label}-{}", + &Uuid::new_v4().to_string()[..8] + )) + .bind(user_id) + .bind(&file_hash) + .bind(data.len() as i64) + .fetch_one(pool) + .await + .expect("file row"); + (file_hash, chunks, file_id) + } + + async fn blob_ref(pool: &PgPool, hash: &str) -> Option { + sqlx::query_scalar("SELECT ref_count FROM storage.blobs WHERE hash = $1") + .bind(hash) + .fetch_optional(pool) + .await + .expect("blob query") + } + + async fn cleanup(pool: &PgPool, file_hash: &str, file_id: Uuid, extra_hashes: &[String]) { + let chunks: Option> = sqlx::query_scalar( + "SELECT chunk_hashes FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(file_hash) + .fetch_optional(pool) + .await + .unwrap_or(None); + let _ = sqlx::query("DELETE FROM storage.files WHERE id = $1") + .bind(file_id) + .execute(pool) + .await; + let _ = sqlx::query("DELETE FROM storage.chunk_manifests WHERE file_hash = $1") + .bind(file_hash) + .execute(pool) + .await; + let mut to_drop = chunks.unwrap_or_default(); + to_drop.push(file_hash.to_string()); + to_drop.extend_from_slice(extra_hashes); + let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = ANY($1)") + .bind(&to_drop) + .execute(pool) + .await; + } + + fn content(len: usize, salt: u8) -> Vec { + let mut data: Vec = (0..len) + .map(|i| { + ((i % 251) as u8) + .wrapping_add(salt) + .wrapping_add((i / 7919) as u8) + }) + .collect(); + data.extend_from_slice(Uuid::new_v4().as_bytes()); + data + } + + // ── Entitlement: claimable vs pin ──────────────────────────── + #[tokio::test] + async fn claim_and_pin_respect_ownership_and_orphans() { + let pool = test_pool().await; + let dir = TempDir::new().unwrap(); + let svc = local_svc(&pool, &dir).await; + let user = seed_user(&pool).await; + + // Owned content (multi-chunk), one foreign chunk (ref 1, no file + // row for this user), one orphan (ref 0), one unknown hash. + let data = content(3 * 1024 * 1024, 21); + let (file_hash, owned_chunks, file_id) = + seed_owned_content(&svc, &pool, user, &data, "claim").await; + assert!(owned_chunks.len() >= 3, "3 MiB must split into ≥3 chunks"); + + let foreign = blake3::hash(format!("foreign-{}", Uuid::new_v4()).as_bytes()) + .to_hex() + .to_string(); + let orphan = blake3::hash(format!("orphan-{}", Uuid::new_v4()).as_bytes()) + .to_hex() + .to_string(); + sqlx::query( + "INSERT INTO storage.blobs (hash, size, ref_count) VALUES ($1, 10, 1), ($2, 10, 0)", + ) + .bind(&foreign) + .bind(&orphan) + .execute(pool.as_ref()) + .await + .expect("seed foreign+orphan"); + let unknown = blake3::hash(format!("unknown-{}", Uuid::new_v4()).as_bytes()) + .to_hex() + .to_string(); + + let mut probe: Vec = owned_chunks.clone(); + probe.push(foreign.clone()); + probe.push(orphan.clone()); + probe.push(unknown.clone()); + + // claimable: only the owned chunks (advisory view — orphans are + // intentionally NOT advertised; the commit pin may still take them). + let claimable = svc.claimable_chunks(user, &probe).await.expect("claimable"); + for c in &owned_chunks { + assert!(claimable.contains(c), "owned chunk {c} must be claimable"); + } + assert!( + !claimable.contains(&foreign), + "foreign chunk must not be claimable" + ); + assert!( + !claimable.contains(&unknown), + "unknown chunk must not be claimable" + ); + + // pin: owned + orphan succeed; foreign and unknown are refused. + let pinned = svc.pin_claimable_chunks(user, &probe).await.expect("pin"); + for c in &owned_chunks { + assert!(pinned.contains(c), "owned chunk {c} must pin"); + } + assert!( + pinned.contains(&orphan), + "ref-0 orphan must pin (just-uploaded state)" + ); + assert!( + !pinned.contains(&foreign), + "foreign owned chunk must NOT pin" + ); + assert!(!pinned.contains(&unknown), "unknown hash must NOT pin"); + + // Ref counts moved exactly where they should. + assert_eq!(blob_ref(&pool, &orphan).await, Some(1), "orphan 0→1"); + assert_eq!( + blob_ref(&pool, &foreign).await, + Some(1), + "foreign untouched" + ); + assert_eq!( + blob_ref(&pool, &owned_chunks[0]).await, + Some(2), + "owned chunk 1→2 (manifest + pin)" + ); + + // Release restores the original counts (clamped at 0). + let pinned_vec: Vec = pinned.into_iter().collect(); + svc.release_pinned_chunks(&pinned_vec).await; + assert_eq!(blob_ref(&pool, &orphan).await, Some(0)); + assert_eq!(blob_ref(&pool, &owned_chunks[0]).await, Some(1)); + + cleanup(&pool, &file_hash, file_id, &[foreign, orphan]).await; + } + + // ── Loose chunk store ──────────────────────────────────────── + #[tokio::test] + async fn loose_chunks_register_as_orphans_without_touching_existing_refs() { + let pool = test_pool().await; + let dir = TempDir::new().unwrap(); + let svc = local_svc(&pool, &dir).await; + let user = seed_user(&pool).await; + + // An owned chunk that the client redundantly re-uploads. + let data = content(100 * 1024, 22); + let (file_hash, owned_chunks, file_id) = + seed_owned_content(&svc, &pool, user, &data, "loose").await; + let owned_chunk_bytes = { + let mut stream = svc.read_blob_stream(&file_hash).await.expect("stream"); + let mut out = Vec::new(); + while let Some(part) = stream.next().await { + out.extend_from_slice(&part.expect("part")); + } + out + }; + + let fresh = content(50 * 1024, 23); + let frames = stream::iter(vec![ + Ok::<_, DomainError>(Bytes::from(fresh.clone())), + Ok(Bytes::from(fresh.clone())), // duplicate frame + Ok(Bytes::from(owned_chunk_bytes.clone())), // already-referenced chunk + ]); + + let received = svc.store_loose_chunks(frames).await.expect("store loose"); + assert_eq!(received.len(), 3, "every frame is answered, in order"); + assert_eq!( + received[0].0, received[1].0, + "duplicate frames share a hash" + ); + let fresh_hash = received[0].0.clone(); + + assert_eq!( + blob_ref(&pool, &fresh_hash).await, + Some(0), + "fresh chunk lands as an unreferenced orphan" + ); + assert_eq!( + blob_ref(&pool, &owned_chunks[0]).await, + Some(1), + "re-uploading an existing chunk must not disturb its refs" + ); + + // The orphan's bytes are really there and addressable. + assert_eq!( + svc.backend().blob_exists(&fresh_hash).await.unwrap(), + true, + "orphan chunk bytes must exist in the backend" + ); + + cleanup(&pool, &file_hash, file_id, &[fresh_hash]).await; + } + + // ── Verification read ──────────────────────────────────────── + #[tokio::test] + async fn hash_chunk_sequence_recomputes_and_validates_sizes() { + let pool = test_pool().await; + let dir = TempDir::new().unwrap(); + let svc = local_svc(&pool, &dir).await; + let user = seed_user(&pool).await; + + let data = content(2 * 1024 * 1024 + 137, 24); + let (file_hash, _chunks, file_id) = + seed_owned_content(&svc, &pool, user, &data, "verify").await; + + let manifest: (Vec, Vec) = sqlx::query_as( + "SELECT chunk_hashes, chunk_sizes FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(&file_hash) + .fetch_one(pool.as_ref()) + .await + .expect("manifest"); + let sequence: Vec<(String, u64)> = manifest + .0 + .iter() + .cloned() + .zip(manifest.1.iter().map(|s| *s as u64)) + .collect(); + + let (computed, head) = svc + .hash_chunk_sequence(&sequence, 16) + .await + .expect("verification read"); + assert_eq!(computed, file_hash, "recomputed hash must match"); + assert_eq!( + &head[..], + &data[..16], + "sniff head must be the file's first bytes" + ); + + // A wrong declared size must be rejected — Range arithmetic + // depends on manifest sizes being true. + let mut lying = sequence.clone(); + lying[0].1 += 1; + assert!( + svc.hash_chunk_sequence(&lying, 0).await.is_err(), + "size lie must fail verification" + ); + + cleanup(&pool, &file_hash, file_id, &[]).await; + } +} diff --git a/src/infrastructure/services/local_blob_backend.rs b/src/infrastructure/services/local_blob_backend.rs index 632701cb..8ef8cf3b 100644 --- a/src/infrastructure/services/local_blob_backend.rs +++ b/src/infrastructure/services/local_blob_backend.rs @@ -244,12 +244,13 @@ impl BlobStorageBackend for LocalBlobBackend { // Atomic rename (same filesystem). Falls back to copy+delete for // cross-device moves (EXDEV errno 18). // - // Durability boundary: the caller is responsible for - // having sync_all'd the source file before invoking this - // function — both `interfaces::upload_spool::spool_body_to_temp` - // and `chunked_upload_service::complete_upload_inner` - // (the two production producers of `source_path`) now do - // so. We fsync the parent of `blob_path` AFTER the rename + // Durability boundary: the caller is responsible for having + // sync_all'd the source file before invoking this function. + // (The streaming upload path writes chunks via + // `put_blob_from_bytes_unsynced` + a batched `sync_blobs` + // sweep instead; this move-based entry point remains for + // whole-file producers such as migration tooling and tests.) + // We fsync the parent of `blob_path` AFTER the rename // so the dirent change itself becomes durable; without // that, a power loss can resurrect the old (unrenamed) // name even when the file contents survive. diff --git a/src/infrastructure/services/nextcloud_chunked_upload_service.rs b/src/infrastructure/services/nextcloud_chunked_upload_service.rs index 34e6cb52..fff15b5e 100644 --- a/src/infrastructure/services/nextcloud_chunked_upload_service.rs +++ b/src/infrastructure/services/nextcloud_chunked_upload_service.rs @@ -73,7 +73,7 @@ impl NextcloudChunkedUploadService { /// Store a chunk in the session directory. Buffers `data` in memory — /// use [`safe_chunk_path`](Self::safe_chunk_path) + the - /// `interfaces/upload_spool::stream_body_to_path` helper to stream the + /// `interfaces/upload_ingest::stream_body_to_path` helper to stream the /// HTTP body directly to disk and avoid materialising the whole chunk /// in RAM. pub async fn store_chunk( @@ -93,24 +93,14 @@ impl NextcloudChunkedUploadService { Ok(()) } - /// Assemble all chunks in numeric order into a temp file, computing - /// the BLAKE3 of the concatenated stream **during** the same read/ - /// write pass (hash-on-write). + /// List the session's chunk files in assembly (numeric) order. /// - /// Returns `(temp_path, total_size, blake3_hex)`. The caller passes - /// the hash to the upload service as `pre_computed_hash` so the - /// downstream dedup layer never has to re-read the assembled file - /// to compute it — saving one full file-sized read pass per upload. - /// - /// The read/hash/write loop runs inside `spawn_blocking` because - /// BLAKE3 is CPU-bound and would otherwise starve the Tokio worker - /// running other connections; synchronous I/O is used inside the - /// blocking thread because the workload is sequential and the - /// async reactor overhead would only slow it down. For files larger - /// than ~10 MB BLAKE3's Rayon mode parallelises across cores — - /// mirrors what `ChunkedUploadService::complete_upload_inner` does - /// for the REST chunked path. - pub async fn assemble(&self, user: &str, upload_id: &str) -> Result<(PathBuf, u64, String)> { + /// The caller streams these directly into the CDC chunk store + /// (`interfaces::upload_ingest::stream_from_files`) — chunking, BLAKE3 + /// hashing and dedup checks happen in that single read pass, so no + /// assembled temp file is ever written. The chunk parts stay on disk + /// until [`cleanup`](Self::cleanup), keeping completion retryable. + pub async fn ordered_chunk_paths(&self, user: &str, upload_id: &str) -> Result> { let session_dir = self.safe_session_dir(user, upload_id)?; let mut entries: Vec = Vec::new(); @@ -124,8 +114,11 @@ impl NextcloudChunkedUploadService { .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))? { let name = entry.file_name().to_string_lossy().to_string(); - if name == ".file" { - continue; // Skip the assembly marker. + // `.file` is the NC-protocol assembly marker; `.assembled` is + // the staging file older releases wrote — sessions in flight + // across an upgrade may still contain one. + if name == ".file" || name == ".assembled" { + continue; } entries.push(name); } @@ -133,74 +126,7 @@ impl NextcloudChunkedUploadService { // Sort chunks numerically (Nextcloud sends them as "00001", "00002", ...). entries.sort(); - let temp_path = session_dir.join(".assembled"); - let chunk_paths: Vec = entries.iter().map(|n| session_dir.join(n)).collect(); - let assembled_for_blocking = temp_path.clone(); - - // Read/hash/write loop runs synchronously on the blocking pool. - // BLAKE3 is computed in the same pass that copies bytes from chunk - // files into the assembled file — no second read after the fact. - let (total_size, hash) = - tokio::task::spawn_blocking(move || -> std::io::Result<(u64, String)> { - use std::io::{BufWriter as StdBufWriter, Read, Write}; - - let raw_output = std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&assembled_for_blocking)?; - - // 512 KB write buffer — 8× fewer syscalls than 64 KB. - let mut output = StdBufWriter::with_capacity(524_288, raw_output); - let mut hasher = blake3::Hasher::new(); - let mut buf = vec![0u8; 524_288]; - let mut total: u64 = 0; - - // Files >10 MB benefit from BLAKE3's multi-threaded mode. - // The threshold matches the REST chunked path's heuristic. - const RAYON_THRESHOLD_PER_FRAME: usize = 128 * 1024; - - for chunk_path in &chunk_paths { - let mut chunk_file = std::fs::File::open(chunk_path)?; - loop { - let n = chunk_file.read(&mut buf)?; - if n == 0 { - break; - } - if n >= RAYON_THRESHOLD_PER_FRAME { - hasher.update_rayon(&buf[..n]); - } else { - hasher.update(&buf[..n]); - } - output.write_all(&buf[..n])?; - total += n as u64; - } - } - - output.flush()?; - // ── Durability boundary ───────────────────────────────── - // sync_all is the actual fsync; without it, a power loss - // before the kernel writeback timer (~5 s) loses - // acknowledged data. Pull the inner File out of the - // BufWriter so we can sync the underlying handle — - // dropping the BufWriter wouldn't trigger fsync. macOS - // caveat: fsync there flushes to the disk controller - // only; true durability needs F_FULLFSYNC, not exposed - // by std. - let raw_output = output - .into_inner() - .map_err(|e| std::io::Error::other(format!("into_inner: {e}")))?; - raw_output.sync_all()?; - - Ok((total, hasher.finalize().to_hex().to_string())) - }) - .await - .map_err(|e| { - DomainError::internal_error("ChunkedUpload", format!("assemble task: {e}")) - })? - .map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?; - - Ok((temp_path, total_size, hash)) + Ok(entries.iter().map(|n| session_dir.join(n)).collect()) } /// Delete the upload session directory. @@ -260,9 +186,10 @@ impl NextcloudChunkedUploadService { let name = entry.file_name().to_string_lossy().to_string(); // Filter internal markers — `.file` is the NC-protocol // assembly trigger target (it never reaches the disk - // because MOVE redirects it), `.assembled` is our own - // staging file from `assemble()`. Surfacing either to - // the client would confuse its chunk-count check. + // because MOVE redirects it), `.assembled` is the staging + // file pre-streaming releases wrote (kept for sessions in + // flight across an upgrade). Surfacing either to the + // client would confuse its chunk-count check. if name == ".file" || name == ".assembled" { continue; } @@ -331,8 +258,18 @@ mod tests { assert!(!svc.session_exists("alice", "upload-999").await); } + /// Concatenate the session's chunk files in assembly order — mirrors + /// what `upload_ingest::stream_from_files` feeds the CDC store. + async fn concat_chunks(svc: &NextcloudChunkedUploadService, user: &str, id: &str) -> Vec { + let mut out = Vec::new(); + for path in svc.ordered_chunk_paths(user, id).await.unwrap() { + out.extend_from_slice(&fs::read(&path).await.unwrap()); + } + out + } + #[tokio::test] - async fn test_store_and_assemble_chunks() { + async fn test_store_and_order_chunks() { let (svc, _dir) = test_service(); svc.create_session("alice", "upload-002").await.unwrap(); @@ -343,20 +280,14 @@ mod tests { .await .unwrap(); - let (temp_path, size, hash) = svc.assemble("alice", "upload-002").await.unwrap(); - let assembled = fs::read(&temp_path).await.unwrap(); - assert_eq!(assembled, b"Hello, World!"); - assert_eq!(size, 13); - // BLAKE3("Hello, World!") — proves hash-on-write happens during - // the assemble pass, not via a re-read. assert_eq!( - hash, - "288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8" + concat_chunks(&svc, "alice", "upload-002").await, + b"Hello, World!" ); } #[tokio::test] - async fn test_assemble_chunks_in_sorted_order() { + async fn test_chunk_paths_sorted_regardless_of_upload_order() { let (svc, _dir) = test_service(); svc.create_session("alice", "upload-003").await.unwrap(); @@ -371,16 +302,29 @@ mod tests { .await .unwrap(); - let (temp_path, size, hash) = svc.assemble("alice", "upload-003").await.unwrap(); - let assembled = fs::read(&temp_path).await.unwrap(); - assert_eq!(assembled, b"ABC"); - assert_eq!(size, 3); - // BLAKE3("ABC") — confirms sort happened (chunks were stored in - // order 3,1,2 but the hash matches "ABC", not "CAB" or "BAC"). - assert_eq!( - hash, - "d1717274597cf0289694f75d96d444b992a096f1afd8e7bbfa6ebb1d360fedfc" - ); + // Chunks were stored in order 3,1,2 but must concatenate as "ABC". + assert_eq!(concat_chunks(&svc, "alice", "upload-003").await, b"ABC"); + } + + #[tokio::test] + async fn test_internal_markers_excluded_from_chunk_paths() { + let (svc, _dir) = test_service(); + svc.create_session("alice", "upload-005").await.unwrap(); + + svc.store_chunk("alice", "upload-005", "00001", b"data") + .await + .unwrap(); + // Stale staging file from a pre-streaming release. + svc.store_chunk("alice", "upload-005", ".assembled", b"old") + .await + .unwrap(); + + let paths = svc + .ordered_chunk_paths("alice", "upload-005") + .await + .unwrap(); + assert_eq!(paths.len(), 1, "markers must be filtered out"); + assert!(paths[0].ends_with("00001")); } #[tokio::test] diff --git a/src/infrastructure/services/trash_cleanup_service.rs b/src/infrastructure/services/trash_cleanup_service.rs index 96f62d2d..b4dde9cf 100644 --- a/src/infrastructure/services/trash_cleanup_service.rs +++ b/src/infrastructure/services/trash_cleanup_service.rs @@ -6,21 +6,35 @@ use tracing::{debug, error, info, instrument}; use crate::common::errors::Result; use crate::domain::repositories::trash_repository::TrashRepository; use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository; +use crate::infrastructure::services::dedup_service::DedupService; /// Service for automatic cleanup of expired items in the trash. /// /// Uses `TrashRepository::delete_expired_bulk` to purge all expired items /// in **2 SQL statements inside a single transaction**, instead of the /// previous N+1 pattern that issued 3 queries per expired item. +/// +/// Each sweep ends with a dedup `garbage_collect()` pass: it reclaims the +/// blobs the expiry just dereferenced AND any other zero-reference rows — +/// notably chunks left behind by aborted streaming uploads, whose rollback +/// registers them at ref_count 0 precisely so this sweep can find them. +/// Without it, orphans would only be collected when a user happens to +/// empty their trash by hand. pub struct TrashCleanupService { trash_repository: Arc, + dedup_service: Arc, cleanup_interval_hours: u64, } impl TrashCleanupService { - pub fn new(trash_repository: Arc, cleanup_interval_hours: u64) -> Self { + pub fn new( + trash_repository: Arc, + dedup_service: Arc, + cleanup_interval_hours: u64, + ) -> Self { Self { trash_repository, + dedup_service, cleanup_interval_hours: cleanup_interval_hours.max(1), // Minimum 1 hour } } @@ -29,6 +43,7 @@ impl TrashCleanupService { #[instrument(skip(self))] pub async fn start_cleanup_job(&self) { let trash_repository = self.trash_repository.clone(); + let dedup_service = self.dedup_service.clone(); let interval_hours = self.cleanup_interval_hours; info!( @@ -41,7 +56,7 @@ impl TrashCleanupService { let mut interval = time::interval(interval_duration); // First immediate execution - Self::cleanup_expired_items(trash_repository.clone()) + Self::cleanup_expired_items(trash_repository.clone(), dedup_service.clone()) .await .unwrap_or_else(|e| error!("Error in initial trash cleanup: {:?}", e)); @@ -49,16 +64,24 @@ impl TrashCleanupService { interval.tick().await; debug!("Running scheduled trash cleanup task"); - if let Err(e) = Self::cleanup_expired_items(trash_repository.clone()).await { + if let Err(e) = + Self::cleanup_expired_items(trash_repository.clone(), dedup_service.clone()) + .await + { error!("Error in scheduled trash cleanup: {:?}", e); } } }); } - /// Bulk-delete all expired trash items in a single transaction. - #[instrument(skip(trash_repository))] - async fn cleanup_expired_items(trash_repository: Arc) -> Result<()> { + /// Bulk-delete all expired trash items in a single transaction, then + /// garbage-collect every zero-reference manifest/blob (expired content + /// plus aborted-upload orphans). + #[instrument(skip(trash_repository, dedup_service))] + async fn cleanup_expired_items( + trash_repository: Arc, + dedup_service: Arc, + ) -> Result<()> { debug!("Starting bulk cleanup of expired trash items"); let (files, folders) = trash_repository.delete_expired_bulk().await?; @@ -72,6 +95,16 @@ impl TrashCleanupService { ); } + // Runs on the maintenance pool; batched (500 rows/iteration) with + // yield points, so it never starves request-path queries. + match dedup_service.garbage_collect().await { + Ok((0, _)) => debug!("Trash cleanup GC: nothing to collect"), + Ok((items, bytes)) => { + info!("Trash cleanup GC: reclaimed {items} orphaned blobs ({bytes} bytes)"); + } + Err(e) => error!("Trash cleanup GC failed: {:?}", e), + } + Ok(()) } } diff --git a/src/infrastructure/services/zip_service.rs b/src/infrastructure/services/zip_service.rs index ed7fff13..a05886c4 100644 --- a/src/infrastructure/services/zip_service.rs +++ b/src/infrastructure/services/zip_service.rs @@ -47,12 +47,41 @@ impl From for DomainError { /// Type alias for the fully-async ZIP writer backed by a buffered tokio file. type AsyncZipWriter = ZipFileWriter>>; +/// One planned archive entry, in final ZIP order. +enum ZipPlanEntry { + /// Directory entry (Stored, zero-length body). + Dir(String), + /// File entry: ZIP-relative path + file id to stream from the blob store. + File { zip_path: String, file_id: String }, +} + +/// Message protocol from the prefetch task to the ZIP writer. For each +/// planned file, in order: zero or more `Chunk`s, then exactly one `End`; +/// `Err` aborts the whole archive. +enum Prefetched { + Chunk(bytes::Bytes), + End, + Err(String), +} + +/// Bound on the prefetch channel (messages of ≤ ~64 KB blob-stream chunks): +/// ~4 MiB of read-ahead. Enough to hide the per-file open latency of the +/// blob store (PG lookup + backend round-trip — significant on S3/Azure) +/// behind the deflate of the previous entry, while keeping RAM flat. +const PREFETCH_BUFFER_CHUNKS: usize = 64; + /// Service for creating ZIP files. /// /// Uses `async_zip` for fully-async archive creation. Every write (headers, /// compressed chunk data, central directory) goes through /// `tokio::io::BufWriter` → `tokio::fs::File`, so **no Tokio worker is ever /// blocked** by disk I/O or compression. +/// +/// Archive creation is a 2-stage pipeline: a prefetch task reads file +/// content from the blob store ahead of the writer, so the next file's +/// read latency overlaps the current file's compression instead of adding +/// to it. The ZIP entries themselves are still written strictly in order +/// (the format requires it). pub struct ZipService { file_service: Arc, folder_service: Arc, @@ -144,7 +173,22 @@ impl ZipService { } }; - // ── 4. Open the temp file + ZIP writer ─────────────────────────── + // ── 4. Plan the archive (folders are already sorted by path) ───── + let mut plan: Vec = Vec::new(); + for folder in &all_folders { + let zip_dir = format!("{}/", folder_zip_path(&folder.path)); + plan.push(ZipPlanEntry::Dir(zip_dir.clone())); + if let Some(files) = files_by_folder.get(&folder.id) { + for file in files { + plan.push(ZipPlanEntry::File { + zip_path: format!("{}{}", zip_dir, file.name), + file_id: file.id.to_string(), + }); + } + } + } + + // ── 5. Open the temp file + ZIP writer ─────────────────────────── let temp = NamedTempFile::new().map_err(ZipError::IoError)?; let tokio_file = tokio::fs::File::create(temp.path()) .await @@ -152,79 +196,131 @@ impl ZipService { let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file); let mut zip = ZipFileWriter::with_tokio(buf_writer); - // ── 5. Write entries (folders are already sorted by path) ───────── - for folder in &all_folders { - let zip_dir = format!("{}/", folder_zip_path(&folder.path)); + // ── 6. Write entries: 2-stage pipeline ─────────────────────────── + // The prefetch task reads blob streams for the planned files, in + // order, ahead of the writer — the next file's blob-store latency + // overlaps the current file's deflate. If the writer bails out, + // dropping the receiver makes the prefetcher's next send fail and + // it stops on its own. + let file_ids: Vec = plan + .iter() + .filter_map(|entry| match entry { + ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()), + ZipPlanEntry::Dir(_) => None, + }) + .collect(); + let (tx, mut rx) = tokio::sync::mpsc::channel::(PREFETCH_BUFFER_CHUNKS); + let _prefetcher = tokio::spawn(Self::prefetch_files( + self.file_service.clone(), + file_ids, + tx, + )); - // Directory entry (Stored, zero-length body) - let dir_entry = ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored); - match zip.write_entry_whole(dir_entry, &[]).await { - Ok(()) => debug!("Folder added to ZIP: {}", zip_dir), - Err(e) => { - warn!("Could not add folder entry (may already exist): {}", e); + for entry in &plan { + match entry { + ZipPlanEntry::Dir(zip_dir) => { + let dir_entry = + ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored); + match zip.write_entry_whole(dir_entry, &[]).await { + Ok(()) => debug!("Folder added to ZIP: {}", zip_dir), + Err(e) => { + warn!("Could not add folder entry (may already exist): {}", e); + } + } } - } - - // Files belonging to this folder - if let Some(files) = files_by_folder.get(&folder.id) { - for file in files { - self.add_file_to_zip_streamed(&mut zip, file, &zip_dir) - .await?; + ZipPlanEntry::File { zip_path, .. } => { + Self::write_prefetched_file(&mut zip, zip_path, &mut rx).await?; } } } - // ── 6. Finalize ────────────────────────────────────────────────── + // ── 7. Finalize ────────────────────────────────────────────────── let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?; compat_writer.close().await.map_err(ZipError::IoError)?; Ok(temp) } - /// Streams file content in chunks (~64 KB) into an async ZIP entry, - /// keeping peak memory independent of individual file sizes. - async fn add_file_to_zip_streamed( - &self, + /// Prefetch stage: streams each planned file's content from the blob + /// store, in plan order, into the bounded channel. Stops on the first + /// read error (after forwarding it) or when the writer hangs up. + async fn prefetch_files( + file_service: Arc, + file_ids: Vec, + tx: tokio::sync::mpsc::Sender, + ) { + for file_id in file_ids { + let stream = match file_service.get_file_stream(&file_id).await { + Ok(s) => s, + Err(e) => { + error!("Error opening file stream {}: {}", file_id, e); + let _ = tx + .send(Prefetched::Err(format!( + "Error streaming file {}: {}", + file_id, e + ))) + .await; + return; + } + }; + + let mut stream = std::pin::Pin::from(stream); + while let Some(chunk_result) = stream.next().await { + let message = match chunk_result { + Ok(bytes) => Prefetched::Chunk(bytes), + Err(e) => Prefetched::Err(format!("Error streaming file {}: {}", file_id, e)), + }; + let abort = matches!(message, Prefetched::Err(_)); + if tx.send(message).await.is_err() || abort { + return; // writer gone, or fatal read error forwarded + } + } + + if tx.send(Prefetched::End).await.is_err() { + return; // writer gone + } + } + } + + /// Writer stage: drains one file's prefetched chunks into a Deflate + /// ZIP entry. Peak memory stays bounded by the channel, independent + /// of individual file sizes. + async fn write_prefetched_file( zip: &mut AsyncZipWriter, - file: &FileDto, - folder_path: &str, + zip_path: &str, + rx: &mut tokio::sync::mpsc::Receiver, ) -> Result<()> { - let file_path = format!("{}{}", folder_path, file.name); - info!("Adding file to ZIP: {}", file_path); + info!("Adding file to ZIP: {}", zip_path); - let file_id = file.id.to_string(); - - // Open a streaming entry with Deflate compression - let entry = ZipEntryBuilder::new(file_path.clone().into(), Compression::Deflate); + let entry = ZipEntryBuilder::new(zip_path.to_string().into(), Compression::Deflate); let mut entry_writer = zip .write_entry_stream(entry) .await .map_err(ZipError::AsyncZipError)?; - // Stream file contents in chunks instead of loading all into RAM - let stream = match self.file_service.get_file_stream(&file_id).await { - Ok(s) => s, - Err(e) => { - error!("Error opening file stream {}: {}", file_id, e); - // Close the partially-opened entry before returning - let _ = entry_writer.close().await; - return Err(ZipError::FileReadError(format!( - "Error streaming file {}: {}", - file_id, e - )) - .into()); + loop { + match rx.recv().await { + Some(Prefetched::Chunk(bytes)) => { + entry_writer + .write_all(&bytes) + .await + .map_err(ZipError::IoError)?; + } + Some(Prefetched::End) => break, + Some(Prefetched::Err(message)) => { + // Close the partially-written entry before bailing out. + let _ = entry_writer.close().await; + return Err(ZipError::FileReadError(message).into()); + } + None => { + let _ = entry_writer.close().await; + return Err(ZipError::FileReadError(format!( + "Prefetch stage ended unexpectedly while writing {}", + zip_path + )) + .into()); + } } - }; - - // Pin the stream so StreamExt::next() can be called - let mut stream = std::pin::Pin::from(stream); - - while let Some(chunk_result) = stream.next().await { - let bytes = chunk_result.map_err(ZipError::IoError)?; - entry_writer - .write_all(&bytes) - .await - .map_err(ZipError::IoError)?; } // Finalize the entry (writes data descriptor with CRC + sizes) @@ -233,7 +329,7 @@ impl ZipService { .await .map_err(ZipError::AsyncZipError)?; - debug!("File added to ZIP: {}", file_path); + debug!("File added to ZIP: {}", zip_path); Ok(()) } } diff --git a/src/interfaces/api/handlers/chunked_upload_handler.rs b/src/interfaces/api/handlers/chunked_upload_handler.rs index f2276e50..0998803b 100644 --- a/src/interfaces/api/handlers/chunked_upload_handler.rs +++ b/src/interfaces/api/handlers/chunked_upload_handler.rs @@ -4,7 +4,7 @@ //! - POST /api/uploads → Create upload session //! - PATCH /api/uploads/:id → Upload a chunk //! - HEAD /api/uploads/:id → Get upload status -//! - POST /api/uploads/:id/complete → Assemble and finalize +//! - POST /api/uploads/:id/complete → Stream parts into the blob store //! - DELETE /api/uploads/:id → Cancel upload use axum::{ @@ -27,7 +27,7 @@ use crate::common::di::AppState; use crate::domain::services::authorization::Permission; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; -use crate::interfaces::upload_spool::stream_body_to_path; +use crate::interfaces::upload_ingest::{self, stream_body_to_path}; /// Request body for creating an upload session #[derive(Debug, Deserialize, ToSchema)] @@ -64,35 +64,32 @@ pub struct CompleteUploadResponse { /// Optional body for `POST /api/uploads/{id}/complete`. /// /// When the client supplies `checksum`, the server compares it against -/// the assembled file's hash BEFORE promoting the blob to storage — -/// failure aborts the upload atomically (no orphaned blob, no DB row). -/// This is the end-to-end integrity check: per-chunk MD5 proves each -/// chunk arrived intact, but only the final hash catches assembly / -/// promotion bugs and mis-ordered chunks. +/// the streamed content's hash BEFORE the file row is created — failure +/// releases the blob reference and returns 400, with the chunk parts +/// kept on disk for a retry. This is the end-to-end integrity check: +/// per-chunk MD5 proves each chunk arrived intact, but only the final +/// hash catches mis-ordered or corrupted assemblies. /// -/// **`blake3` is highly recommended** — it's the algorithm the server -/// already runs over the assembled file during hash-on-write -/// assembly, so verification is a string comparison with zero extra -/// I/O and zero extra CPU. It's also the same algorithm the server -/// uses for blob-storage addressing, so the value the client sends -/// equals the `content_hash` they'd later read back from -/// `GET /api/files/{id}`. `md5` and `sha256` are accepted for -/// compatibility with legacy client tooling but each triggers a -/// second hash pass over the assembled file (~30–100 ms depending -/// on size). +/// **`blake3` is highly recommended** — it's the content-addressing +/// algorithm of the blob store itself, so verification is a string +/// comparison against the hash the store already computed, and the +/// value the client sends equals the `content_hash` they'd later read +/// back from `GET /api/files/{id}`. `md5` and `sha256` are accepted +/// for legacy client tooling; they are computed by an in-flight tee +/// during the same streaming pass — no extra disk read either way. /// /// `Default` keeps the existing wire shape: clients that POST with no /// body get today's behavior (no verification, server just returns /// what it computed). #[derive(Debug, Default, Deserialize, ToSchema)] pub struct CompleteUploadRequest { - /// Lowercase hex digest the client expects the assembled file to + /// Lowercase hex digest the client expects the streamed content to /// hash to. Compared case-insensitively. Omit to skip verification. pub checksum: Option, /// Algorithm name. `blake3` is the recommended choice (default — - /// matches the server's hash-on-write algorithm, zero extra cost). - /// `md5`, `sha256` / `sha-256` are accepted but trigger an extra - /// hash pass. Unknown values return 400. + /// matches the blob store's content-addressing algorithm). `md5`, + /// `sha256` / `sha-256` are accepted and computed in-flight. + /// Unknown values return 400. pub checksumalg: Option, } @@ -307,70 +304,15 @@ impl ChunkedUploadHandler { } } - /// Compute the requested checksum of the assembled file. - /// - /// For `Blake3` the server already has the hash from hash-on-write - /// assembly — we just return it (zero I/O, zero CPU). For `Md5` and - /// `Sha256` we re-read the assembled file on the blocking pool and - /// hash it; the cost (~30–100 ms for typical files) is the trade-off - /// for accepting non-default algorithms. - async fn compute_assembled_hash( - assembled_path: &std::path::Path, - alg: ChecksumAlg, - blake3_already_computed: &str, - ) -> Result { - match alg { - ChecksumAlg::Blake3 => Ok(blake3_already_computed.to_string()), - ChecksumAlg::Md5 | ChecksumAlg::Sha256 => { - let path = assembled_path.to_path_buf(); - tokio::task::spawn_blocking(move || -> Result { - use std::io::Read; - let mut file = std::fs::File::open(&path)?; - let mut buf = vec![0u8; 524_288]; - match alg { - ChecksumAlg::Md5 => { - use md5::Digest as _; - let mut h = md5::Md5::new(); - loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } - h.update(&buf[..n]); - } - Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect()) - } - ChecksumAlg::Sha256 => { - use sha2::Digest as _; - let mut h = sha2::Sha256::new(); - loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } - h.update(&buf[..n]); - } - Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect()) - } - // Blake3 handled above — this branch is unreachable but - // keeps the match exhaustive without an else-clause. - ChecksumAlg::Blake3 => unreachable!(), - } - }) - .await - .map_err(|e| std::io::Error::other(format!("hash task join failed: {e}")))? - } - } - } - /// POST /api/uploads/:upload_id/complete - Finalize upload /// - /// Assembles all chunks into the final file and creates the file record. - /// When `body.checksum` is supplied, the assembled file's hash is - /// verified before the blob is promoted to storage — mismatch - /// returns 400 and the assembled temp is removed (the session - /// itself is kept so the client can re-issue complete after - /// diagnosing). + /// Streams the uploaded chunk parts, in order, straight into the CDC + /// chunk store and creates the file record — no assembled temp file is + /// ever written. When `body.checksum` is supplied it is verified from + /// the same streaming pass (BLAKE3 comes from the store itself; + /// MD5/SHA-256 are computed by an in-flight tee) — mismatch returns 400 + /// with the blob reference released, and the chunk parts stay on disk + /// so the client can re-issue complete after diagnosing. pub(super) async fn complete_upload_impl( State(state): State>, auth_user: AuthUser, @@ -379,10 +321,11 @@ impl ChunkedUploadHandler { ) -> impl IntoResponse { let chunked_service = &state.core.chunked_upload_service; let upload_service = &state.applications.file_upload_service; + let dedup = &state.core.dedup_service; - // ── Parse the optional algorithm BEFORE assembly so a bad - // `checksumalg` doesn't waste the (potentially expensive) - // hash work on a request we'll reject anyway. + // ── Parse the optional algorithm BEFORE completion so a bad + // `checksumalg` doesn't waste any work on a request we'll + // reject anyway. let alg = match body.checksumalg.as_deref() { Some(name) => match ChecksumAlg::parse(name) { Some(a) => Some(a), @@ -397,37 +340,54 @@ impl ChunkedUploadHandler { }; let expected_checksum = body.checksum.as_deref(); - // Assemble chunks (hash-on-write: BLAKE3 computed during assembly) - let (assembled_path, filename, folder_id, content_type, total_size, hash) = - match chunked_service - .complete_upload(&upload_id, auth_user.id) - .await - { - Ok(result) => result, - Err(e) => { - return AppError::from(e).into_response(); - } - }; + // Validate completion and get the chunk parts in assembly order. + let parts = match chunked_service + .complete_upload(&upload_id, auth_user.id) + .await + { + Ok(result) => result, + Err(e) => { + return AppError::from(e).into_response(); + } + }; + + // MD5/SHA-256 verification taps the stream while it is ingested; + // BLAKE3 needs no tee — the store's own content hash IS BLAKE3. + let alg = expected_checksum.map(|_| alg.unwrap_or(ChecksumAlg::Blake3)); + let tee = match alg { + Some(ChecksumAlg::Md5) | Some(ChecksumAlg::Sha256) => { + Some(upload_ingest::checksum_tee(alg.unwrap())) + } + _ => None, + }; + + // ── Stream the parts into the CDC chunk store ─────────────── + let ingested = match upload_ingest::ingest_stream_to_cas( + upload_ingest::stream_from_files(parts.chunk_paths), + dedup, + &parts.filename, + &parts.content_type, + usize::MAX, + tee.clone(), + ) + .await + { + Ok(ingested) => ingested, + Err(e) => return e.into_response(), + }; // ── End-to-end integrity verification ─────────────────────── - // Only fires when the client supplied an `expected` checksum. - // For BLAKE3 (the documented preferred choice) this is a string - // comparison against the hash assembly already produced. For - // MD5/SHA-256 we re-hash the assembled file on the blocking pool. - if let Some(expected) = expected_checksum { - let alg = alg.unwrap_or(ChecksumAlg::Blake3); - let computed = match Self::compute_assembled_hash(&assembled_path, alg, &hash).await { - Ok(c) => c, - Err(e) => { - let _ = tokio::fs::remove_file(&assembled_path).await; - return AppError::internal_error(format!( - "Failed to compute assembled checksum: {e}" - )) - .into_response(); - } + if let (Some(expected), Some(alg)) = (expected_checksum, alg) { + let computed = match alg { + ChecksumAlg::Blake3 => Some(ingested.hash.clone()), + _ => tee.as_ref().and_then(upload_ingest::finalize_checksum_tee), + }; + let Some(computed) = computed else { + upload_ingest::discard_ingested(dedup, &ingested).await; + return AppError::internal_error("Checksum tee produced no digest").into_response(); }; if !computed.eq_ignore_ascii_case(expected) { - let _ = tokio::fs::remove_file(&assembled_path).await; + upload_ingest::discard_ingested(dedup, &ingested).await; tracing::warn!( target: "audit", event = "chunked_upload.checksum_mismatch", @@ -449,36 +409,28 @@ impl ChunkedUploadHandler { } } - // ── MIME detection (magic bytes + extension fallback) ───── - let content_type = crate::common::mime_detect::refine_content_type_from_file( - &assembled_path, - &filename, - &content_type, - ) - .await; - - // Upload from assembled file on disk — zero extra RAM copies, hash pre-computed + // Register the file row against the ingested blob. + let size = ingested.size; match upload_service - .upload_file_from_path( - filename.clone(), - folder_id.clone(), - content_type, - &assembled_path, - Some(hash), + .upload_file_streaming( + parts.filename.clone(), + parts.folder_id.clone(), + ingested.content_type.clone(), + ingested.stored(), ) .await { Ok(file) => { - // Cleanup session + // Cleanup session (removes the chunk part files) let _ = chunked_service .finalize_upload(&upload_id, auth_user.id) .await; tracing::info!( "✅ CHUNKED UPLOAD COMPLETE: {} (ID: {}, {} bytes)", - filename, + parts.filename, file.id, - total_size + size ); ( @@ -486,14 +438,14 @@ impl ChunkedUploadHandler { Json(CompleteUploadResponse { file_id: file.id, filename: file.name, - size: total_size, + size, path: file.path, }), ) .into_response() } Err(e) => { - tracing::error!("Failed to create file from assembled upload: {:?}", e); + tracing::error!("Failed to create file from chunked upload: {:?}", e); AppError::internal_error(format!("Failed to create file: {}", e)).into_response() } } diff --git a/src/interfaces/api/handlers/dedup_handler.rs b/src/interfaces/api/handlers/dedup_handler.rs index 3defb869..3e2f29db 100644 --- a/src/interfaces/api/handlers/dedup_handler.rs +++ b/src/interfaces/api/handlers/dedup_handler.rs @@ -5,12 +5,11 @@ use axum::{ response::IntoResponse, }; use serde::Serialize; -use tokio::io::AsyncWriteExt; use utoipa::ToSchema; -use crate::application::ports::dedup_ports::DedupResultDto; use crate::common::di::AppState; use crate::interfaces::middleware::auth::AuthUser; +use crate::interfaces::upload_ingest; use std::sync::Arc; /// Global application state for dependency injection @@ -155,10 +154,10 @@ impl DedupHandler { /// Upload content with automatic deduplication (streaming). /// - /// Spools the upload to a temp file while computing the BLAKE3 hash - /// incrementally (hash-on-write). Memory usage is constant (~512 KB) - /// regardless of file size. Then delegates to `store_from_file` with - /// the pre-computed hash so the file is never re-read for hashing. + /// Streams the multipart field straight into the CDC chunk store — + /// chunking, BLAKE3 hashing and dedup checks happen while the bytes + /// arrive (no temp file, no re-read; peak RAM is bounded regardless + /// of file size). /// /// POST /api/dedup/upload pub(super) async fn upload_with_dedup_impl( @@ -177,64 +176,29 @@ impl DedupHandler { .content_type() .unwrap_or("application/octet-stream") .to_string(); + let filename = field.file_name().unwrap_or("unnamed").to_string(); - // ── Spool to temp file + BLAKE3 hash-on-write ──────── - let temp_dir = state.core.path_service.get_root_path().join(".dedup_temp"); - let temp_path = temp_dir.join(format!("dedup-{}", uuid::Uuid::new_v4())); - - let mut total_size: u64 = 0; - let mut hasher = blake3::Hasher::new(); - let mut field = field; - - let spool_result: Result<(), String> = async { - let file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| format!("Failed to create temp file: {}", e))?; - - // 512 KB buffer — reduces write syscalls - let mut writer = tokio::io::BufWriter::with_capacity(524_288, file); - - loop { - match field.chunk().await { - Ok(Some(chunk)) => { - total_size += chunk.len() as u64; - hasher.update(&chunk); - writer - .write_all(&chunk) - .await - .map_err(|e| format!("Failed to write chunk: {}", e))?; - } - Ok(None) => break, - Err(e) => { - return Err(format!( - "Connection lost during upload (received {} bytes): {}", - total_size, e - )); - } - } + // ── Stream into the CDC chunk store ────────────────── + let source = upload_ingest::multipart_field_stream(field); + let ingested = match upload_ingest::ingest_stream_to_cas( + source, + dedup, + &filename, + &content_type, + usize::MAX, + None, + ) + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::warn!("Dedup upload ingest failed: {}", e.message); + return e.into_response(); } + }; - writer - .flush() - .await - .map_err(|e| format!("Failed to flush temp file: {}", e))?; - Ok(()) - } - .await; - - if let Err(msg) = spool_result { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::warn!("Dedup upload spool failed: {}", msg); - return Response::builder() - .status(StatusCode::BAD_REQUEST) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(format!(r#"{{"error": "{}"}}"#, msg))) - .unwrap() - .into_response(); - } - - if total_size == 0 { - let _ = tokio::fs::remove_file(&temp_path).await; + if ingested.size == 0 { + upload_ingest::discard_ingested(dedup, &ingested).await; return Response::builder() .status(StatusCode::BAD_REQUEST) .header(header::CONTENT_TYPE, "application/json") @@ -243,60 +207,33 @@ impl DedupHandler { .into_response(); } - let hash = hasher.finalize().to_hex().to_string(); + let metadata = dedup.get_blob_metadata(&ingested.hash).await; - // ── Store with deduplication (pre-computed hash) ────── - match dedup - .store_from_file(&temp_path, Some(content_type), Some(hash)) - .await - { - Ok(result) => { - let (is_new, bytes_saved) = match &result { - DedupResultDto::NewBlob { .. } => (true, 0), - DedupResultDto::ExistingBlob { saved_bytes, .. } => { - (false, *saved_bytes) - } - }; + let response = DedupUploadResponse { + is_new: ingested.is_new_blob, + hash: ingested.hash.clone(), + size: ingested.size, + bytes_saved: ingested.bytes_saved, + ref_count: metadata.map(|m| m.ref_count).unwrap_or(1), + }; - let metadata = dedup.get_blob_metadata(result.hash()).await; + tracing::info!( + "🔗 Dedup upload: hash={}, new={}, saved={}", + ingested.hash, + ingested.is_new_blob, + ingested.bytes_saved + ); - let response = DedupUploadResponse { - is_new, - hash: result.hash().to_string(), - size: result.size(), - bytes_saved, - ref_count: metadata.map(|m| m.ref_count).unwrap_or(1), - }; - - tracing::info!( - "🔗 Dedup upload: hash={}, new={}, saved={}", - result.hash(), - is_new, - bytes_saved - ); - - return Response::builder() - .status(if is_new { - StatusCode::CREATED - } else { - StatusCode::OK - }) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(serde_json::to_string(&response).unwrap())) - .unwrap() - .into_response(); - } - Err(e) => { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("Dedup upload failed: {}", e); - return Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(r#"{"error": "Upload failed"}"#)) - .unwrap() - .into_response(); - } - } + return Response::builder() + .status(if ingested.is_new_blob { + StatusCode::CREATED + } else { + StatusCode::OK + }) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&response).unwrap())) + .unwrap() + .into_response(); } } diff --git a/src/interfaces/api/handlers/delta_upload_handler.rs b/src/interfaces/api/handlers/delta_upload_handler.rs new file mode 100644 index 00000000..6912d37f --- /dev/null +++ b/src/interfaces/api/handlers/delta_upload_handler.rs @@ -0,0 +1,441 @@ +//! Delta-upload protocol endpoints — "upload only what changed". +//! +//! Wire surface of [`DeltaUploadService`]; see that module (and +//! `docs/delta-upload-protocol.md`) for the protocol and its security +//! model. These handlers only authenticate, rate-limit and translate +//! the wire formats — every decision lives in the application service. +//! +//! Chunk-frame wire format (`PUT /api/files/delta/chunks`): the body is a +//! sequence of `[u32 big-endian length][length bytes]` frames, one per +//! chunk, with `Content-Type: application/octet-stream`. Frames are capped +//! at the CDC maximum chunk size (1 MiB) and the whole request at the same +//! per-request ceiling as resumable chunk PUTs (`chunk_max_bytes`). + +use axum::{ + Json, + body::Body, + extract::{Path, State}, + http::{HeaderMap, StatusCode, header}, + response::{IntoResponse, Response}, +}; +use bytes::{Buf, Bytes, BytesMut}; +use futures::Stream; +use std::sync::Arc; +use tokio_stream::StreamExt; + +use crate::application::services::delta_upload_service::{ + DeltaChunksResponse, DeltaCommitOutcome, DeltaCommitRequest, DeltaDownloadOutcome, + DeltaDownloadRequest, DeltaManifestResponse, DeltaNegotiateRequest, DeltaNegotiateResponse, +}; +use crate::common::di::AppState; +use crate::common::errors::DomainError; +use crate::infrastructure::services::dedup_service::CDC_MAX_CHUNK; +use crate::interfaces::errors::AppError; +use crate::interfaces::middleware::auth::AuthUser; +use http_body_util::BodyStream; +use serde::Serialize; +use utoipa::ToSchema; + +/// 409 body of `POST /api/files/delta/commit` when chunks vanished between +/// negotiate and commit (GC race) or were never claimable: the client +/// uploads exactly these hashes and retries the same commit. +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaStillMissingResponse { + pub still_missing: Vec, +} + +/// 404 body of `POST /api/files/delta/download` when some requested chunks +/// are not reachable through the caller's files (or don't exist — the two +/// are deliberately indistinguishable). +#[derive(Debug, Serialize, ToSchema)] +pub struct DeltaNotAvailableResponse { + pub not_available: Vec, +} + +/// Per-caller flood guard shared by the delta endpoints. +fn check_rate_limit(state: &Arc, auth_user: &AuthUser) -> Result<(), AppError> { + if state + .delta_upload_rate_limiter + .check_and_increment(&auth_user.id.to_string()) + .is_err() + { + tracing::info!( + target: "audit", + event = "delta_upload.rejected", + reason = "rate_limited", + caller_id = %auth_user.id, + "👮🏻‍♂️ Delta upload rejected: per-caller rate limit exceeded", + ); + return Err(AppError::new( + StatusCode::TOO_MANY_REQUESTS, + "Too many delta-upload requests; please retry shortly", + "RateLimited", + )); + } + Ok(()) +} + +/// Parse a `[u32 BE length][bytes]` frame sequence from the request body. +/// +/// Streaming: peak RAM is one frame (≤ 1 MiB) plus one HTTP frame, +/// regardless of how many chunks the request carries. `max_total` bounds +/// the whole request body. +fn parse_chunk_frames( + body: Body, + max_total: usize, +) -> impl Stream> + Send { + async_stream::try_stream! { + let mut body_stream = BodyStream::new(body); + let mut buf = BytesMut::new(); + let mut expecting: Option = None; + let mut total: usize = 0; + + loop { + // Drain every complete frame already buffered. + loop { + match expecting { + None => { + if buf.len() < 4 { + break; + } + let len = + u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize; + buf.advance(4); + if len == 0 || len > CDC_MAX_CHUNK { + Err(DomainError::validation_error(format!( + "Chunk frame of {len} bytes out of bounds (1 ..= {CDC_MAX_CHUNK})" + )))?; + } + expecting = Some(len); + } + Some(len) => { + if buf.len() < len { + break; + } + let frame = buf.split_to(len).freeze(); + expecting = None; + yield frame; + } + } + } + + match body_stream.next().await { + Some(Ok(http_frame)) => { + if let Some(data) = http_frame.data_ref() { + total += data.len(); + if total > max_total { + Err(DomainError::validation_error(format!( + "Request body exceeds the {max_total}-byte per-request cap; \ + split the chunk upload into several requests" + )))?; + } + buf.extend_from_slice(data); + } + } + Some(Err(e)) => { + Err(DomainError::validation_error(format!( + "Failed to read request body: {e}" + )))?; + } + None => { + if expecting.is_some() || !buf.is_empty() { + Err(DomainError::validation_error( + "Truncated chunk frame at end of body", + ))?; + } + break; + } + } + } + } +} + +#[utoipa::path( + post, + path = "/api/files/delta/negotiate", + request_body = DeltaNegotiateRequest, + responses( + (status = 200, description = "Chunks the caller must upload (the rest are claimable without bytes)", body = DeltaNegotiateResponse), + (status = 400, description = "Malformed chunk list (hash format, size bounds, count ceiling)"), + (status = 429, description = "Rate limited"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_negotiate( + State(state): State>, + auth_user: AuthUser, + Json(request): Json, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let response = state + .applications + .delta_upload_service + .negotiate_with_perms(auth_user.id, &request) + .await + .map_err(AppError::from)?; + Ok(Json(response)) +} + +#[utoipa::path( + put, + path = "/api/files/delta/chunks", + request_body(content_type = "application/octet-stream", + description = "Sequence of [u32 BE length][bytes] frames, one per chunk (each ≤ 1 MiB)"), + responses( + (status = 200, description = "Server-computed identity of every received frame, in order", body = DeltaChunksResponse), + (status = 400, description = "Malformed framing, oversized frame, or oversized request"), + (status = 429, description = "Rate limited"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_upload_chunks( + State(state): State>, + auth_user: AuthUser, + body: Body, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let frames = parse_chunk_frames(body, state.core.config.storage.chunk_max_bytes); + let response = state + .applications + .delta_upload_service + .receive_chunks(frames) + .await + .map_err(AppError::from)?; + Ok(Json(response)) +} + +#[utoipa::path( + post, + path = "/api/files/delta/commit", + request_body = DeltaCommitRequest, + responses( + (status = 201, description = "File created from the committed chunk sequence", body = crate::application::dtos::file_dto::FileDto), + (status = 200, description = "Existing file's content replaced", body = crate::application::dtos::file_dto::FileDto), + (status = 400, description = "Malformed request, or the declared file_hash does not match the chunk sequence"), + (status = 404, description = "Target folder/file not found or not accessible"), + (status = 409, description = "Chunks not claimable — upload them and retry", body = DeltaStillMissingResponse), + (status = 429, description = "Rate limited"), + (status = 507, description = "Storage quota exceeded"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_commit( + State(state): State>, + auth_user: AuthUser, + Json(request): Json, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let outcome = state + .applications + .delta_upload_service + .commit_with_perms(auth_user.id, request) + .await + .map_err(AppError::from)?; + Ok(match outcome { + DeltaCommitOutcome::Done { file, created } => { + let status = if created { + StatusCode::CREATED + } else { + StatusCode::OK + }; + (status, Json(file)).into_response() + } + DeltaCommitOutcome::StillMissing(still_missing) => ( + StatusCode::CONFLICT, + Json(DeltaStillMissingResponse { still_missing }), + ) + .into_response(), + }) +} + +#[utoipa::path( + get, + path = "/api/files/{id}/manifest", + params(("id" = String, Path, description = "File ID")), + responses( + (status = 200, description = "Chunk recipe of the file (immutable per file_hash; served with ETag = file_hash)", body = DeltaManifestResponse), + (status = 304, description = "Not modified (If-None-Match matched the current file_hash)"), + (status = 404, description = "File not found, not accessible, or not owned by the caller"), + (status = 429, description = "Rate limited"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_file_manifest( + State(state): State>, + auth_user: AuthUser, + Path(file_id): Path, + headers: HeaderMap, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let manifest = state + .applications + .delta_upload_service + .file_manifest_with_perms(auth_user.id, &file_id) + .await + .map_err(AppError::from)?; + + // A manifest is immutable for a given file_hash, so the hash IS the + // strong validator: sync clients polling a file revalidate for free. + let etag = format!("\"{}\"", manifest.file_hash); + if headers + .get(header::IF_NONE_MATCH) + .and_then(|v| v.to_str().ok()) + .is_some_and(|inm| inm == etag) + { + return Ok(Response::builder() + .status(StatusCode::NOT_MODIFIED) + .header(header::ETAG, etag) + .body(Body::empty()) + .unwrap()); + } + Ok(( + StatusCode::OK, + [ + (header::ETAG, etag), + (header::CACHE_CONTROL, "private, no-cache".to_string()), + ], + Json(manifest), + ) + .into_response()) +} + +#[utoipa::path( + post, + path = "/api/files/delta/download", + request_body = DeltaDownloadRequest, + responses( + (status = 200, description = "Requested chunks as [u32 BE length][bytes] frames, in request order", + content_type = "application/octet-stream"), + (status = 400, description = "Malformed hashes, duplicates, or batch above the per-request ceiling"), + (status = 404, description = "Some chunks are not available to this caller", body = DeltaNotAvailableResponse), + (status = 429, description = "Rate limited"), + ), + security(("bearerAuth" = [])), + tag = "delta-upload" +)] +pub async fn delta_download_chunks( + State(state): State>, + auth_user: AuthUser, + Json(request): Json, +) -> Result { + check_rate_limit(&state, &auth_user)?; + let service = state.applications.delta_upload_service.clone(); + let outcome = service + .authorize_chunk_download_with_perms(auth_user.id, &request) + .await + .map_err(AppError::from)?; + + let ordered = match outcome { + DeltaDownloadOutcome::NotAvailable(not_available) => { + return Ok(( + StatusCode::NOT_FOUND, + Json(DeltaNotAvailableResponse { not_available }), + ) + .into_response()); + } + DeltaDownloadOutcome::Ready(ordered) => ordered, + }; + let total: u64 = ordered.iter().map(|(_, s)| 4 + s).sum(); + + // Stream the frames: 4-byte length headers come from the (entitled) + // index sizes; bytes stream straight from the blob backend. Peak RAM + // is one backend read frame, independent of batch size. + let body_stream: std::pin::Pin> + Send>> = + Box::pin(async_stream::try_stream! { + for (hash, size) in ordered { + yield Bytes::copy_from_slice(&(size as u32).to_be_bytes()); + let mut chunk = service.chunk_stream(&hash).await.map_err(std::io::Error::other)?; + while let Some(part) = chunk.next().await { + yield part?; + } + } + }); + Ok(Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/octet-stream") + .header(header::CONTENT_LENGTH, total.to_string()) + .body(Body::from_stream(body_stream)) + .unwrap()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Encode frames the way a client would. + fn encode(frames: &[&[u8]]) -> Vec { + let mut out = Vec::new(); + for f in frames { + out.extend_from_slice(&(f.len() as u32).to_be_bytes()); + out.extend_from_slice(f); + } + out + } + + async fn collect(body: Body, max_total: usize) -> Result, DomainError> { + let stream = parse_chunk_frames(body, max_total); + futures::pin_mut!(stream); + let mut out = Vec::new(); + while let Some(item) = stream.next().await { + out.push(item?); + } + Ok(out) + } + + #[tokio::test] + async fn roundtrips_frames_in_order() { + let wire = encode(&[b"first", b"second chunk", &[0xAB; 1000]]); + let frames = collect(Body::from(wire), usize::MAX).await.unwrap(); + assert_eq!(frames.len(), 3); + assert_eq!(&frames[0][..], b"first"); + assert_eq!(&frames[1][..], b"second chunk"); + assert_eq!(frames[2].len(), 1000); + } + + #[tokio::test] + async fn empty_body_yields_no_frames() { + let frames = collect(Body::empty(), usize::MAX).await.unwrap(); + assert!(frames.is_empty()); + } + + #[tokio::test] + async fn rejects_zero_length_frame() { + let wire = encode(&[b""]); + assert!(collect(Body::from(wire), usize::MAX).await.is_err()); + } + + #[tokio::test] + async fn rejects_frame_above_cdc_max() { + let mut wire = Vec::new(); + wire.extend_from_slice(&((CDC_MAX_CHUNK as u32) + 1).to_be_bytes()); + // Header alone is enough — the length is rejected before any data. + assert!(collect(Body::from(wire), usize::MAX).await.is_err()); + } + + #[tokio::test] + async fn rejects_truncated_frame() { + let mut wire = encode(&[b"complete"]); + wire.extend_from_slice(&10u32.to_be_bytes()); + wire.extend_from_slice(b"only5"); // promises 10, delivers 5 + assert!(collect(Body::from(wire), usize::MAX).await.is_err()); + } + + #[tokio::test] + async fn rejects_request_above_total_cap() { + let wire = encode(&[&[0u8; 600], &[1u8; 600]]); + assert!(collect(Body::from(wire), 1000).await.is_err()); + } + + #[tokio::test] + async fn accepts_frame_exactly_at_cdc_max() { + let big = vec![7u8; CDC_MAX_CHUNK]; + let wire = encode(&[&big]); + let frames = collect(Body::from(wire), usize::MAX).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].len(), CDC_MAX_CHUNK); + } +} diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index d80cd494..67c27f57 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -21,6 +21,7 @@ use crate::common::di::AppState; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::AuthUser; use crate::interfaces::range_requests::not_modified_response; +use crate::interfaces::upload_ingest; use crate::{application::dtos::file_dto::FileDto, domain::services::authorization::Permission}; use std::sync::Arc; @@ -51,11 +52,12 @@ impl FileHandler { // UPLOAD // ═══════════════════════════════════════════════════════════════════════ - /// Streaming file upload — constant ~64 KB RAM regardless of file size. + /// Streaming file upload — bounded RAM regardless of file size. /// - /// **Hash-on-Write**: BLAKE3 is computed while spooling the multipart - /// body to the temp file. This eliminates the second sequential read - /// that dedup_service would otherwise need, cutting total I/O in half. + /// The multipart body is streamed straight into the CDC chunk store: + /// chunking, hashing and dedup checks happen while the bytes arrive. + /// No spool file, no re-read — chunks the store already has are never + /// written to disk at all. pub async fn upload_file( State(state): State, auth_user: AuthUser, @@ -67,11 +69,62 @@ impl FileHandler { } } + /// Instant upload: create a file from a blob the caller already owns. + /// + /// Zero content bytes travel — the client proved possession of the + /// content by hash (it computed BLAKE3 locally and confirmed via + /// `GET /api/dedup/check/{hash}`), so the server only bumps the blob's + /// reference count and registers the metadata row. + /// + /// All authorization (folder Create permission, hash ownership with + /// anti-enumeration, quota) lives in the application service. + pub(super) async fn create_file_by_hash_impl( + State(state): State, + auth_user: AuthUser, + Json(request): Json, + ) -> impl IntoResponse { + // Hash shape check — same contract as /api/dedup/check/{hash}. + if request.hash.len() != 64 || !request.hash.chars().all(|c| c.is_ascii_hexdigit()) { + return AppError::bad_request( + "Invalid hash format. Expected BLAKE3 (64 hex characters)", + ) + .into_response(); + } + // Basename only — same path-traversal guard as the multipart upload. + let filename = request + .name + .rsplit('/') + .next() + .unwrap_or(&request.name) + .rsplit('\\') + .next() + .unwrap_or(&request.name) + .to_string(); + if filename.is_empty() { + return AppError::bad_request("File name must not be empty").into_response(); + } + + match state + .applications + .file_upload_service + .create_file_from_owned_blob_with_perms( + auth_user.id, + filename, + request.folder_id, + &request.hash, + ) + .await + { + Ok(file) => Self::created_json_response(&file).into_response(), + Err(err) => Self::domain_error_response(err).into_response(), + } + } + /// Core upload logic shared by [`Self::upload_file`] and /// [`Self::upload_file_with_thumbnails`]. /// /// Returns `(FileDto, blob_hash)` on success. The blob hash is the - /// BLAKE3 digest computed during the hash-on-write spool and is + /// BLAKE3 digest computed during the streaming ingest and is /// propagated without an extra database round-trip so that callers /// (e.g. thumbnail generation) can resolve the physical blob path /// immediately. @@ -158,120 +211,55 @@ impl FileHandler { } } - // ── Spool multipart field to temp file + hash-on-write ── - // .dedup_temp is created once by DedupService::initialize() at startup - let temp_dir = state.core.path_service.get_root_path().join(".dedup_temp"); - let temp_path = temp_dir.join(format!("upload-{}", uuid::Uuid::new_v4())); - - let mut total_size: u64 = 0; - let mut hasher = blake3::Hasher::new(); - let spool_result: Result<(), String> = async { - let file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| format!("Failed to create temp file: {}", e))?; - - // Pre-allocate if Content-Length is known (reduces fragmentation) - let hint = field - .headers() - .get(axum::http::header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()); - if let Some(len) = hint { - let _ = file.set_len(len).await; // best-effort - } - - // 512 KB buffer — 8× fewer write syscalls than 64 KB - let mut writer = tokio::io::BufWriter::with_capacity(524_288, file); - let mut field = field; - // IMPORTANT: use explicit match instead of `while let Ok(Some(..))`. - // The old pattern silently swallowed Err (client disconnect) - // and accepted partially received data as a complete upload. - loop { - match field.chunk().await { - Ok(Some(chunk)) => { - total_size += chunk.len() as u64; - hasher.update(&chunk); - tokio::io::AsyncWriteExt::write_all(&mut writer, &chunk) - .await - .map_err(|e| format!("Failed to write chunk: {}", e))?; - } - Ok(None) => break, // End of field — upload complete - Err(e) => { - return Err(format!( - "Connection lost during upload (received {} bytes): {}", - total_size, e - )); - } - } - } - tokio::io::AsyncWriteExt::flush(&mut writer) - .await - .map_err(|e| format!("Failed to flush temp file: {}", e))?; - Ok(()) - } - .await; - - if let Err(e) = spool_result { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("❌ UPLOAD SPOOL FAILED: {} - {}", filename, e); - return Err(Self::domain_error_response( - crate::common::errors::DomainError::internal_error("FileUpload", e), - )); - } - - // Empty file — use streaming path with the (empty) temp file - if total_size == 0 { - let hash = hasher.finalize().to_hex().to_string(); - let dto = upload_service - .upload_file_streaming( - filename, - folder_id, - content_type, - &temp_path, - 0, - Some(hash.clone()), - ) - .await - .map_err(Self::domain_error_response)?; - return Ok((dto, hash)); - } - - // Finalize hash - let hash = hasher.finalize().to_hex().to_string(); - - // ── MIME detection (magic bytes + extension fallback) ─ - let content_type = crate::common::mime_detect::refine_content_type_from_file( - &temp_path, + // ── Stream the field into the CDC chunk store ──────── + // Chunking (FastCDC) + hashing (BLAKE3) + dedup checks + + // MIME sniffing all happen while the bytes arrive; chunks + // the store already has never touch the disk. Size is + // capped globally by DefaultBodyLimit. + let dedup = &state.core.dedup_service; + let source = upload_ingest::multipart_field_stream(field); + let ingested = match upload_ingest::ingest_stream_to_cas( + source, + dedup, &filename, &content_type, + usize::MAX, + None, ) - .await; + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::error!("❌ UPLOAD INGEST FAILED: {} - {}", filename, e.message); + return Err(e.into_response()); + } + }; - // ── Quota enforcement ──────────────────────────────── + // ── Quota enforcement (exact size now known) ───────── if let Some(storage_svc) = state.storage_usage_service.as_ref() && let Err(err) = storage_svc - .check_storage_quota(auth_user.id, total_size) + .check_storage_quota(auth_user.id, ingested.size) .await { - let _ = tokio::fs::remove_file(&temp_path).await; + upload_ingest::discard_ingested(dedup, &ingested).await; tracing::warn!( "⛔ UPLOAD REJECTED (quota): user={}, file={}, size={}", auth_user.username, filename, - total_size + ingested.size ); return Err(Self::quota_error_response(err)); } - // ── Streaming upload (temp file → blob store, hash pre-computed) ─ + // ── Register the file row against the ingested blob ── + let hash = ingested.hash.clone(); + let size = ingested.size; match upload_service .upload_file_streaming( filename.clone(), folder_id, - content_type, - &temp_path, - total_size, - Some(hash.clone()), + ingested.content_type.clone(), + ingested.stored(), ) .await { @@ -279,13 +267,12 @@ impl FileHandler { tracing::info!( "✅ STREAMING UPLOAD: {} ({} bytes, ID: {})", filename, - total_size, + size, file.id ); return Ok((file, hash)); } Err(err) => { - let _ = tokio::fs::remove_file(&temp_path).await; tracing::error!("❌ UPLOAD FAILED: {} - {}", filename, err); return Err(Self::domain_error_response(err)); } @@ -1103,6 +1090,39 @@ pub async fn upload_file_with_thumbnails( FileHandler::upload_file_with_thumbnails_impl(state, auth_user, multipart).await } +/// Request body for the instant-upload endpoint. +#[derive(Debug, Deserialize, ToSchema)] +pub struct CreateFileByHashRequest { + /// File name to create (path components are stripped). + pub name: String, + /// Target folder ID (the caller needs Create permission on it). + pub folder_id: String, + /// BLAKE3 hash (64 hex chars) of content the caller already owns. + pub hash: String, +} + +#[utoipa::path( + post, + path = "/api/files/by-hash", + request_body = CreateFileByHashRequest, + responses( + (status = 201, description = "File created from an already-owned blob — zero bytes transferred", body = FileDto), + (status = 400, description = "Invalid hash format or empty name"), + (status = 404, description = "No owned blob with this hash (anti-enumeration: same shape as unknown hash)"), + (status = 409, description = "A file with this name already exists in the folder"), + (status = 507, description = "Storage quota exceeded"), + ), + security(("bearerAuth" = [])), + tag = "files" +)] +pub async fn create_file_by_hash( + state: State, + auth_user: AuthUser, + request: Json, +) -> impl IntoResponse { + FileHandler::create_file_by_hash_impl(state, auth_user, request).await +} + #[utoipa::path( get, path = "/api/files/{id}", diff --git a/src/interfaces/api/handlers/mod.rs b/src/interfaces/api/handlers/mod.rs index 2033ed2c..aebeb707 100644 --- a/src/interfaces/api/handlers/mod.rs +++ b/src/interfaces/api/handlers/mod.rs @@ -7,6 +7,7 @@ pub mod carddav_handler; pub mod chunked_upload_handler; pub mod contacts_handler; pub mod dedup_handler; +pub mod delta_upload_handler; pub mod device_auth_handler; pub mod favorites_handler; pub mod file_handler; diff --git a/src/interfaces/api/handlers/webdav_handler.rs b/src/interfaces/api/handlers/webdav_handler.rs index c0097eb3..4b8c2117 100644 --- a/src/interfaces/api/handlers/webdav_handler.rs +++ b/src/interfaces/api/handlers/webdav_handler.rs @@ -898,10 +898,11 @@ async fn handle_head( /** * Handles PUT requests to create or update files. * - * **Streaming implementation**: the request body is spooled to a temp file - * with incremental BLAKE3 hashing. Peak RAM usage is ~256 KB regardless - * of file size. The temp file is then atomically moved into blob storage - * via `update_file_streaming`. + * **Streaming implementation**: the request body is streamed straight into + * the CDC chunk store (FastCDC + BLAKE3 while the bytes arrive — no spool + * file, no re-read; peak RAM is bounded regardless of file size), then the + * file row is atomically swapped onto the ingested blob via + * `update_file_streaming`. * * @param state The application state containing service dependencies * @param path The requested resource path @@ -913,7 +914,7 @@ async fn handle_put( req: Request, path: String, ) -> Result, AppError> { - use crate::interfaces::upload_spool::spool_body_to_temp; + use crate::interfaces::upload_ingest; let user = extract_user(&req)?; @@ -972,32 +973,31 @@ async fn handle_put( .unwrap_or("application/octet-stream") .to_string(); - // ── Streaming spool: body → temp file + incremental hash ── - // Shared with the NextCloud-compat PUT handler; peak heap ~one frame - // regardless of file size. Honors `upload_temp_dir` to keep the spool - // off tmpfs/RAM. - let spooled = spool_body_to_temp( + // ── Streaming ingest: body → CDC chunk store ────────────── + // Shared with the NextCloud-compat PUT handler; chunking + hashing + + // dedup checks run while the body arrives — no spool file, no re-read. + let filename = crate::common::mime_detect::filename_from_path(&path).to_string(); + let ingested = upload_ingest::ingest_body_to_cas( req.into_body(), + &state.core.dedup_service, + &filename, + &content_type, max_upload, - state.core.config.storage.upload_temp_dir.clone(), ) .await?; - let temp_path = spooled.temp.path().to_path_buf(); - let total_bytes = spooled.size as usize; - let hash = spooled.hash; // ── Quota enforcement ──────────────────────────────────── if let Some(storage_svc) = state.storage_usage_service.as_ref() && let Err(err) = storage_svc - .check_storage_quota(user.id, total_bytes as u64) + .check_storage_quota(user.id, ingested.size) .await { - let _ = tokio::fs::remove_file(&temp_path).await; + upload_ingest::discard_ingested(&state.core.dedup_service, &ingested).await; tracing::warn!( "⛔ WEBDAV PUT REJECTED (quota): user={}, file={}, size={}", user.id, path, - total_bytes + ingested.size ); return Err(AppError::new( StatusCode::INSUFFICIENT_STORAGE, @@ -1006,21 +1006,12 @@ async fn handle_put( )); } - // ── Atomic store: temp file → dedup blob + DB metadata update ── + // ── Atomic store: swap the file row onto the ingested blob ── + let content_type = ingested.content_type.clone(); let result = file_upload_service - .update_file_streaming( - &path, - &temp_path, - total_bytes as u64, - &content_type, - Some(hash), - None, - ) + .update_file_streaming(&path, ingested.stored(), &content_type, None) .await; - // Clean up temp file (may already be moved by dedup, ignore error) - let _ = tokio::fs::remove_file(&temp_path).await; - match result { Ok(_file_dto) => Ok(Response::builder() .status(StatusCode::NO_CONTENT) diff --git a/src/interfaces/api/handlers/wopi_handler.rs b/src/interfaces/api/handlers/wopi_handler.rs index 80ed7b81..93e3320b 100644 --- a/src/interfaces/api/handlers/wopi_handler.rs +++ b/src/interfaces/api/handlers/wopi_handler.rs @@ -165,10 +165,6 @@ async fn put_file( State(state): State, req: Request, ) -> Response { - use http_body_util::BodyStream; - use tokio::io::AsyncWriteExt; - use tokio_stream::StreamExt; - let claims = match state .token_service .validate_token(&token_query.access_token) @@ -218,75 +214,32 @@ async fn put_file( Err(_) => return StatusCode::NOT_FOUND.into_response(), }; - // ── Streaming spool: body → temp file + incremental BLAKE3 ── - let temp_file = match tempfile::NamedTempFile::new() { - Ok(f) => f, - Err(e) => { - tracing::error!("WOPI PutFile: failed to create temp file: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - let temp_path = temp_file.path().to_path_buf(); - - let mut file_out = match tokio::fs::File::create(&temp_path).await { - Ok(f) => f, - Err(e) => { - tracing::error!("WOPI PutFile: failed to open temp file: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - + // ── Streaming ingest: body → CDC chunk store (no temp file) ── let content_type = file.mime_type.clone(); - let mut hasher = blake3::Hasher::new(); - let mut total_bytes: u64 = 0; - let mut stream = BodyStream::new(req.into_body()); - - while let Some(frame_result) = stream.next().await { - let frame = match frame_result { - Ok(f) => f, - Err(e) => { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: body read error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len() as u64; - hasher.update(chunk); - if let Err(e) = file_out.write_all(chunk).await { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: temp write error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } + let ingested = match crate::interfaces::upload_ingest::ingest_body_to_cas( + req.into_body(), + &state.app_state.core.dedup_service, + &file.name, + &content_type, + usize::MAX, + ) + .await + { + Ok(ingested) => ingested, + Err(e) => { + tracing::error!("WOPI PutFile: ingest failed: {}", e.message); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - } - if let Err(e) = file_out.flush().await { - let _ = tokio::fs::remove_file(&temp_path).await; - tracing::error!("WOPI PutFile: flush error: {}", e); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - drop(file_out); + }; - let hash = hasher.finalize().to_hex().to_string(); - - // ── Atomic store: temp file → dedup blob + DB metadata update ── + // ── Atomic store: swap the file row onto the ingested blob ── let result = state .app_state .applications .file_upload_service - .update_file_streaming( - &file.path, - &temp_path, - total_bytes, - &content_type, - Some(hash), - None, - ) + .update_file_streaming(&file.path, ingested.stored(), &content_type, None) .await; - // Clean up temp file (may already be moved by dedup, ignore error) - let _ = tokio::fs::remove_file(&temp_path).await; - match result { Ok(_file_dto) => StatusCode::OK.into_response(), Err(e) => { diff --git a/src/interfaces/api/mod.rs b/src/interfaces/api/mod.rs index e725fc67..e30113f3 100644 --- a/src/interfaces/api/mod.rs +++ b/src/interfaces/api/mod.rs @@ -80,6 +80,12 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; // File handlers (free functions — see file_handler.rs for why) handlers::file_handler::list_files_query, handlers::file_handler::upload_file_with_thumbnails, + handlers::file_handler::create_file_by_hash, + handlers::delta_upload_handler::delta_negotiate, + handlers::delta_upload_handler::delta_upload_chunks, + handlers::delta_upload_handler::delta_commit, + handlers::delta_upload_handler::delta_file_manifest, + handlers::delta_upload_handler::delta_download_chunks, handlers::file_handler::download_file, handlers::file_handler::get_thumbnail, handlers::file_handler::upload_thumbnail, @@ -261,6 +267,16 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; ResourceContentDto, // File schemas FileDto, + // Delta-upload schemas + crate::application::services::delta_upload_service::ChunkRef, + crate::application::services::delta_upload_service::DeltaNegotiateRequest, + crate::application::services::delta_upload_service::DeltaNegotiateResponse, + crate::application::services::delta_upload_service::DeltaChunksResponse, + crate::application::services::delta_upload_service::DeltaCommitRequest, + handlers::delta_upload_handler::DeltaStillMissingResponse, + handlers::delta_upload_handler::DeltaNotAvailableResponse, + crate::application::services::delta_upload_service::DeltaManifestResponse, + crate::application::services::delta_upload_service::DeltaDownloadRequest, MoveFilePayload, PaginationDto, PaginationRequestDto, diff --git a/src/interfaces/api/routes.rs b/src/interfaces/api/routes.rs index f2ac1662..14c9d017 100644 --- a/src/interfaces/api/routes.rs +++ b/src/interfaces/api/routes.rs @@ -54,9 +54,12 @@ use crate::interfaces::api::handlers::batch_handler::{self, BatchHandlerState}; use crate::interfaces::api::handlers::chunked_upload_handler::{ cancel_upload, complete_upload, create_upload, get_upload_status, upload_chunk, }; +use crate::interfaces::api::handlers::delta_upload_handler::{ + delta_commit, delta_download_chunks, delta_file_manifest, delta_negotiate, delta_upload_chunks, +}; use crate::interfaces::api::handlers::file_handler::{ - delete_file, download_file, get_file_metadata, get_thumbnail, list_files_query, - move_file_simple, rename_file, upload_file_with_thumbnails, upload_thumbnail, + create_file_by_hash, delete_file, download_file, get_file_metadata, get_thumbnail, + list_files_query, move_file_simple, rename_file, upload_file_with_thumbnails, upload_thumbnail, }; #[allow(deprecated)] use crate::interfaces::api::handlers::folder_handler::{ @@ -229,6 +232,12 @@ pub fn create_api_routes(app_state: &Arc) -> Router> { let basic_file_router = Router::new() .route("/", get(list_files_query)) .route("/upload", post(upload_file_with_thumbnails)) + .route("/by-hash", post(create_file_by_hash)) + .route("/delta/negotiate", post(delta_negotiate)) + .route("/delta/chunks", put(delta_upload_chunks)) + .route("/delta/commit", post(delta_commit)) + .route("/delta/download", post(delta_download_chunks)) + .route("/{id}/manifest", get(delta_file_manifest)) .route("/{id}", get(download_file)) .route( "/{id}/thumbnail/{size}", diff --git a/src/interfaces/mod.rs b/src/interfaces/mod.rs index fef37883..f9c9f41f 100644 --- a/src/interfaces/mod.rs +++ b/src/interfaces/mod.rs @@ -3,7 +3,7 @@ pub mod errors; pub mod middleware; pub mod nextcloud; pub mod range_requests; -pub mod upload_spool; +pub mod upload_ingest; pub mod web; pub use api::create_api_routes; diff --git a/src/interfaces/nextcloud/uploads_handler.rs b/src/interfaces/nextcloud/uploads_handler.rs index 8bb6ca8f..16dd5b07 100644 --- a/src/interfaces/nextcloud/uploads_handler.rs +++ b/src/interfaces/nextcloud/uploads_handler.rs @@ -7,10 +7,12 @@ use std::sync::Arc; use crate::application::ports::file_ports::{FileRetrievalUseCase, FileUploadUseCase}; use crate::common::di::AppState; -use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file}; +use crate::common::mime_detect::filename_from_path; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; -use crate::interfaces::upload_spool::stream_body_to_path; +use crate::interfaces::upload_ingest::{ + discard_ingested, ingest_stream_to_cas, stream_body_to_path, stream_from_files, +}; /// Dispatch Nextcloud chunked upload WebDAV requests. /// @@ -239,18 +241,17 @@ async fn handle_assemble( let dest_subpath = extract_files_subpath(&destination, &user.username) .ok_or_else(|| AppError::bad_request("Invalid Destination URL"))?; - // Assemble chunks into a temp file with hash-on-write (BLAKE3 computed - // during the same read/write loop that copies chunks into the - // assembled file). The hash is passed downstream as `pre_computed_hash` - // so the dedup layer never re-reads the assembled file just to compute - // it — saves one full file-sized read pass per upload. - let (temp_path, size, blake3_hash) = nc + // Stream the chunk parts, in order, straight into the CDC chunk store — + // no assembled temp file is ever written. Chunking (FastCDC), BLAKE3 + // hashing, dedup checks and MIME sniffing (magic bytes off the first + // part) all happen in that single read pass. The parts stay on disk + // until the session cleanup below, so a failed completion is retryable. + let chunk_paths = nc .chunked_uploads - .assemble(&user.username, upload_id) + .ordered_chunk_paths(&user.username, upload_id) .await - .map_err(|e| AppError::internal_error(format!("Failed to assemble chunks: {}", e)))?; + .map_err(|e| AppError::internal_error(format!("Failed to list chunks: {}", e)))?; - // Write assembled file to storage via the upload service. let upload_service = &state.applications.file_upload_service; let file_service = &state.applications.file_retrieval_service; let folder_service = &state.applications.folder_service; @@ -261,35 +262,31 @@ async fn handle_assemble( dest_subpath.trim_matches('/') ); - // Detect content type via magic bytes + extension fallback. - let filename = filename_from_path(&dest_subpath); - let content_type = - refine_content_type_from_file(&temp_path, filename, "application/octet-stream").await; + let filename = filename_from_path(&dest_subpath).to_string(); + let ingested = ingest_stream_to_cas( + stream_from_files(chunk_paths), + &state.core.dedup_service, + &filename, + "application/octet-stream", + usize::MAX, + None, + ) + .await?; + let content_type = ingested.content_type.clone(); // Check if file exists (update vs create). let existing = file_service.get_file_by_path(&internal_path).await; let etag: Option = if existing.is_ok() { let dto = upload_service - .update_file_streaming( - &internal_path, - &temp_path, - size, - &content_type, - Some(blake3_hash.clone()), - oc_mtime, - ) + .update_file_streaming(&internal_path, ingested.stored(), &content_type, oc_mtime) .await .map_err(|e| AppError::internal_error(format!("Failed to update file: {}", e)))?; Some(dto.etag) } else { - // New-file branch: resolve the parent folder by path and pass the - // assembled file's path directly to `upload_file_from_path` so the - // bytes never get read back into RAM. Previously this branch did - // `tokio::fs::read(&temp_path)` — an extra full file-sized read - // pass AND a peak-RAM allocation equal to the upload size, which - // defeated the streaming model on large NC uploads. + // New-file branch: resolve the parent folder by path and register + // the file row against the already-ingested blob. let (parent_sub, filename) = match dest_subpath.rsplit_once('/') { Some((p, n)) => (p, n), None => ("", dest_subpath.as_str()), @@ -302,18 +299,23 @@ async fn handle_assemble( let parent_internal = parent_internal.trim_end_matches('/'); use crate::application::ports::folder_ports::FolderUseCase; - let parent_folder = folder_service - .get_folder_by_path(parent_internal) - .await - .map_err(|e| AppError::internal_error(format!("Parent folder lookup failed: {}", e)))?; + let parent_folder = match folder_service.get_folder_by_path(parent_internal).await { + Ok(folder) => folder, + Err(e) => { + discard_ingested(&state.core.dedup_service, &ingested).await; + return Err(AppError::internal_error(format!( + "Parent folder lookup failed: {}", + e + ))); + } + }; let dto = upload_service - .upload_file_from_path( + .upload_file_streaming( filename.to_string(), Some(parent_folder.id), content_type.to_string(), - &temp_path, - Some(blake3_hash), + ingested.stored(), ) .await .map_err(|e| AppError::internal_error(format!("Failed to create file: {}", e)))?; @@ -321,9 +323,6 @@ async fn handle_assemble( Some(dto.etag) }; - // Clean up temp file (session cleanup below removes the directory anyway). - let _ = tokio::fs::remove_file(&temp_path).await; - // Cleanup session. let _ = nc.chunked_uploads.cleanup(&user.username, upload_id).await; diff --git a/src/interfaces/nextcloud/webdav_handler.rs b/src/interfaces/nextcloud/webdav_handler.rs index 8999fe04..e74aff48 100644 --- a/src/interfaces/nextcloud/webdav_handler.rs +++ b/src/interfaces/nextcloud/webdav_handler.rs @@ -22,12 +22,12 @@ use crate::application::ports::file_ports::{ use crate::application::ports::folder_ports::FolderUseCase; use crate::application::ports::trash_ports::TrashUseCase; use crate::common::di::AppState; -use crate::common::mime_detect::{filename_from_path, refine_content_type_from_file}; +use crate::common::mime_detect::filename_from_path; use crate::interfaces::api::handlers::webdav_handler::PROPFIND_BATCH_SIZE; use crate::interfaces::errors::AppError; use crate::interfaces::middleware::auth::{AuthUser, CurrentUser}; use crate::interfaces::range_requests::{not_modified_response, range_response}; -use crate::interfaces::upload_spool::spool_body_to_temp; +use crate::interfaces::upload_ingest::ingest_body_to_cas; /// Extension trait to map XML write errors to `String` concisely. trait XmlResultExt { @@ -575,46 +575,35 @@ async fn handle_put( // at 95 % loses everything. let max_upload = state.core.config.storage.direct_put_max_bytes; - // Stream the body to a temp file + incremental hash — never buffer the - // full upload in RAM. The old `body::to_bytes` path loaded the entire - // file (e.g. an 800 MB ISO) into anonymous memory before any dedup logic, - // OOMKilling the process even on dedup hits. Shared with the native - // WebDAV PUT handler; peak heap ~one frame regardless of file size. - let spooled = spool_body_to_temp( - req.into_body(), - max_upload, - state.core.config.storage.upload_temp_dir.clone(), - ) - .await?; - - // Detect real MIME type from the first bytes on disk (no full read). + // Stream the body straight into the CDC chunk store — never buffer the + // full upload in RAM and never spool it to disk. Chunking, hashing, + // dedup checks and MIME sniffing (magic bytes off the first frames) + // all run while the body arrives; chunks the store already has are + // never written at all. Shared with the native WebDAV PUT handler. + // // `filename` is owned so we don't hold a borrow of the `subpath` param // across the await (which would make the handler future non-Send). let filename = filename_from_path(subpath).to_string(); - let content_type = - refine_content_type_from_file(spooled.temp.path(), &filename, &claimed_type).await; + let ingested = ingest_body_to_cas( + req.into_body(), + &state.core.dedup_service, + &filename, + &claimed_type, + max_upload, + ) + .await?; + let content_type = ingested.content_type.clone(); // Distinguish create (201) vs update (204) for the response status. let existed = file_service.get_file_by_path(&internal_path).await.is_ok(); // Single streaming path — handles both update and create internally, - // passing the precomputed hash so the dedup fast path can short-circuit - // without re-reading the file. + // swapping the file row onto the already-ingested blob. let stored = upload_service - .update_file_streaming( - &internal_path, - spooled.temp.path(), - spooled.size, - &content_type, - Some(spooled.hash), - oc_mtime, - ) + .update_file_streaming(&internal_path, ingested.stored(), &content_type, oc_mtime) .await .map_err(|e| AppError::internal_error(format!("Failed to store file: {}", e)))?; - // dedup may have already moved the temp on a new-blob store; ignore error. - let _ = tokio::fs::remove_file(spooled.temp.path()).await; - let status = if existed { StatusCode::NO_CONTENT } else { diff --git a/src/interfaces/upload_ingest.rs b/src/interfaces/upload_ingest.rs new file mode 100644 index 00000000..c1ecf8a8 --- /dev/null +++ b/src/interfaces/upload_ingest.rs @@ -0,0 +1,588 @@ +//! Shared streaming upload ingestion: request body → CDC chunk store. +//! +//! Used by every upload surface (REST multipart, native WebDAV PUT, +//! NextCloud PUT, chunked-upload assembly, WOPI PutFile) so none of them +//! buffers the full body in RAM **or spools it to a temp file**. The bytes +//! flow straight into [`DedupService::store_from_stream`], which chunks +//! (FastCDC), hashes (BLAKE3) and dedup-checks them while they arrive — +//! each uploaded byte touches the disk at most once, and not at all when +//! the store already has its chunk. +//! +//! MIME refinement happens in-flight: when the claimed Content-Type is +//! generic, the first bytes are peeked off the stream for magic-byte +//! detection before being forwarded unchanged. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::Mutex as StdMutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +use axum::body::Body; +use bytes::Bytes; +use futures::stream::{self, Stream, StreamExt, TryStreamExt}; +use http_body_util::BodyStream; +// The `Digest` trait (re-exported by both `md5` and `sha2` from the +// `digest` crate) gives `Md5` and `Sha256` their `new` / `update` / +// `finalize` methods. Importing once via `sha2` covers both — +// otherwise every call site would need fully-qualified +// `::…` syntax. +use sha2::Digest as _; +use tokio::io::AsyncWriteExt; +use tokio_util::io::ReaderStream; + +use crate::application::ports::chunked_upload_ports::ChecksumAlg; +use crate::application::ports::file_ports::StoredBlob; +use crate::common::mime_detect::{MAGIC_BYTES_LEN, is_generic_mime, refine_content_type}; +use crate::infrastructure::services::dedup_service::DedupService; +use crate::interfaces::errors::AppError; + +/// Content stored in the chunk store by one upload ingest. +/// +/// The ingest holds ONE blob reference; pass [`IngestedBlob::stored`] to the +/// upload service (which takes ownership of the reference) or hand it back +/// via [`discard_ingested`] when the upload is rejected after the fact. +pub struct IngestedBlob { + /// BLAKE3 of the full content (the blob/manifest key). + pub hash: String, + /// Total bytes ingested. + pub size: u64, + /// Refined content type (claimed type or magic-byte detection). + pub content_type: String, + /// `false` when the exact content already existed (dedup hit). + pub is_new_blob: bool, + /// Bytes that did not need to be transferred to storage (dedup hit). + pub bytes_saved: u64, +} + +impl IngestedBlob { + /// The blob reference to hand to the upload service. + pub fn stored(&self) -> StoredBlob { + StoredBlob { + hash: self.hash.clone(), + size: self.size, + is_new_blob: self.is_new_blob, + } + } +} + +/// Hand back the blob reference taken by a successful ingest when the upload +/// is rejected after the fact (quota exceeded, checksum mismatch, …). +pub async fn discard_ingested(dedup: &DedupService, blob: &IngestedBlob) { + if let Err(e) = dedup.remove_reference(&blob.hash).await { + tracing::warn!( + "Failed to release blob reference of rejected upload {}: {e}", + &blob.hash[..blob.hash.len().min(12)] + ); + } +} + +/// Shared mutable tee for computing a client-requested checksum during the +/// ingest pass (REST chunked uploads) — no post-store re-read needed. +pub type ChecksumTee = Arc>>; + +/// Create a checksum tee for [`ingest_stream_to_cas`]. +pub fn checksum_tee(alg: ChecksumAlg) -> ChecksumTee { + Arc::new(StdMutex::new(Some(IncrementalHasher::new(alg)))) +} + +/// Finalize a checksum tee into its lowercase hex digest. +pub fn finalize_checksum_tee(tee: &ChecksumTee) -> Option { + tee.lock() + .ok() + .and_then(|mut h| h.take()) + .map(IncrementalHasher::finalize_hex) +} + +/// Out-of-band state observed by the stream adapters while the dedup engine +/// consumes the stream — lets the caller map an opaque engine error back to +/// the precise HTTP failure (413 vs 400). +struct IngestFlags { + too_large: AtomicBool, + source_error: StdMutex>, +} + +/// Stream a request body (or any byte stream) into the CDC chunk store. +/// +/// Single pass: size-cap enforcement, optional checksum tee, MIME sniffing +/// (first [`MAGIC_BYTES_LEN`] bytes, only when `claimed_type` is generic) +/// and the CDC chunk/hash/store pipeline all run while the bytes arrive. +/// Peak heap is bounded by the dedup engine (~9 MiB) regardless of size. +/// +/// On error nothing stays referenced — the engine compensates internally. +pub async fn ingest_stream_to_cas( + source: S, + dedup: &Arc, + filename: &str, + claimed_type: &str, + max_bytes: usize, + checksum: Option, +) -> Result +where + S: Stream> + Send, + E: std::fmt::Display, +{ + let flags = Arc::new(IngestFlags { + too_large: AtomicBool::new(false), + source_error: StdMutex::new(None), + }); + + // ── Adapter: cap + checksum tee + error capture ────────────── + let adapter_flags = flags.clone(); + let mut total: usize = 0; + let counted = source.map(move |item| match item { + Ok(bytes) => { + total += bytes.len(); + if total > max_bytes { + adapter_flags.too_large.store(true, Ordering::Relaxed); + return Err(std::io::Error::other("upload exceeds size cap")); + } + if let Some(tee) = &checksum + && let Ok(mut hasher) = tee.lock() + && let Some(hasher) = hasher.as_mut() + { + hasher.update(&bytes); + } + Ok(bytes) + } + Err(e) => { + let message = e.to_string(); + if let Ok(mut slot) = adapter_flags.source_error.lock() { + *slot = Some(message.clone()); + } + Err(std::io::Error::other(message)) + } + }); + // `fuse` is load-bearing: when the source is shorter than the MIME peek + // (< MAGIC_BYTES_LEN), the peek loop drains it to None and the `chain` + // below polls it once more — non-fused sources (e.g. `stream::unfold`, + // as used for multipart fields) panic on a post-None poll. + let mut counted = Box::pin(counted.fuse()); + + // ── In-flight MIME sniff (only when the claimed type is generic) ── + let mut head: Vec> = Vec::new(); + let content_type = if is_generic_mime(claimed_type) { + let mut head_len = 0usize; + while head_len < MAGIC_BYTES_LEN { + match counted.next().await { + Some(Ok(bytes)) => { + head_len += bytes.len(); + head.push(Ok(bytes)); + } + Some(Err(e)) => { + head.push(Err(e)); + break; + } + None => break, + } + } + let mut magic = Vec::with_capacity(head_len.min(MAGIC_BYTES_LEN)); + for item in head.iter().flatten() { + let take = (MAGIC_BYTES_LEN - magic.len()).min(item.len()); + magic.extend_from_slice(&item[..take]); + if magic.len() >= MAGIC_BYTES_LEN { + break; + } + } + refine_content_type(&magic, filename, claimed_type) + } else { + claimed_type.to_string() + }; + + // ── Store: peeked head + remainder, one continuous stream ──── + let full_stream = stream::iter(head).chain(counted); + let result = dedup + .store_from_stream(full_stream, Some(content_type.clone())) + .await; + + match result { + Ok(stored) => { + let is_new_blob = !stored.was_deduplicated(); + let bytes_saved = match &stored { + crate::application::ports::dedup_ports::DedupResultDto::ExistingBlob { + saved_bytes, + .. + } => *saved_bytes, + _ => 0, + }; + Ok(IngestedBlob { + hash: stored.hash().to_string(), + size: stored.size(), + content_type, + is_new_blob, + bytes_saved, + }) + } + Err(e) => { + if flags.too_large.load(Ordering::Relaxed) { + return Err(AppError::payload_too_large(format!( + "Upload body exceeds the direct-PUT cap ({max_bytes} bytes). \ + Use the chunked-upload protocol (REST: `/api/uploads/...`, \ + NextCloud: `/remote.php/dav/uploads/...`) for files larger than this. \ + Chunked uploads are resumable on transient failure." + ))); + } + let source_error = flags.source_error.lock().ok().and_then(|s| s.clone()); + if let Some(message) = source_error { + return Err(AppError::bad_request(format!( + "Failed to read request body: {message}" + ))); + } + Err(AppError::from(e)) + } + } +} + +/// [`ingest_stream_to_cas`] for an HTTP request body. +pub async fn ingest_body_to_cas( + body: Body, + dedup: &Arc, + filename: &str, + claimed_type: &str, + max_bytes: usize, +) -> Result { + let source = BodyStream::new(body).filter_map(|item| async move { + match item { + Ok(frame) => frame.into_data().ok().map(Ok), + Err(e) => Some(Err(e)), + } + }); + ingest_stream_to_cas(source, dedup, filename, claimed_type, max_bytes, None).await +} + +/// Adapt a multipart field into a byte stream for [`ingest_stream_to_cas`]. +/// +/// Terminates after the first error — multipart fields are not resumable. +pub fn multipart_field_stream( + field: axum::extract::multipart::Field<'_>, +) -> impl Stream> + Send + '_ { + stream::unfold((field, false), |(mut field, done)| async move { + if done { + return None; + } + match field.chunk().await { + Ok(Some(bytes)) => Some((Ok(bytes), (field, false))), + Ok(None) => None, + Err(e) => Some((Err(e), (field, true))), + } + }) +} + +/// Concatenate already-uploaded chunk part files into one byte stream, in +/// the given order — feeds chunked-upload assembly into the CDC store +/// without ever materializing an assembled file on disk. +pub fn stream_from_files( + paths: Vec, +) -> impl Stream> + Send { + stream::iter(paths.into_iter().map(Ok::<_, std::io::Error>)) + .and_then(|path| async move { + tokio::fs::File::open(path) + .await + .map(|file| ReaderStream::with_capacity(file, 64 * 1024)) + }) + .try_flatten() +} + +/// Result of a streamed write to a caller-supplied path. +pub struct StreamedToPath { + /// Total bytes written. + pub bytes_written: u64, + /// Lowercase hex digest, populated only when `checksum_alg=Some(_)` + /// was passed. The algorithm is identified by [`StreamedToPath::alg`]. + pub checksum_hex: Option, + /// Algorithm used to compute `checksum_hex`. Echoed back so the + /// caller can include it in audit logs or response headers. + pub alg: Option, +} + +/// Stream an HTTP request body directly to a known destination file, +/// enforcing `max_bytes` as a hard size limit. +/// +/// Used by the chunked-upload PUT handlers — each chunk has a +/// deterministic on-disk path (`NextcloudChunkedUploadService::safe_chunk_path` +/// for the NC surface, `ChunkedUploadService::prepare_chunk` for the +/// REST surface), so there's no spool/move dance. Peak heap is ~one +/// HTTP frame regardless of chunk size or `max_bytes`. +/// +/// `checksum_alg` is the optional client-requested integrity check +/// (default `md5` per the legacy `Content-MD5` contract; `blake3` +/// available for forward-compat). When `Some`, the hash is computed +/// incrementally during streaming — no extra disk read for verification. +/// +/// On size overflow the partial file is removed before the function +/// returns, so a client retry against the same chunk name starts from +/// a clean slate. On any other I/O error the partial file is also +/// removed and the error surfaces — callers can assume the path is +/// either fully written or absent. +pub async fn stream_body_to_path( + body: Body, + path: &Path, + max_bytes: usize, + checksum_alg: Option, +) -> Result { + let mut file = tokio::fs::File::create(path) + .await + .map_err(|e| AppError::internal_error(format!("Failed to open chunk file: {e}")))?; + + let mut total_bytes: usize = 0; + let mut stream = BodyStream::new(body); + let mut hasher = checksum_alg.map(IncrementalHasher::new); + + while let Some(frame_result) = stream.next().await { + let frame = match frame_result { + Ok(f) => f, + Err(e) => { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::bad_request(format!( + "Failed to read request body: {e}" + ))); + } + }; + if let Some(chunk) = frame.data_ref() { + total_bytes += chunk.len(); + if total_bytes > max_bytes { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::payload_too_large(format!( + "Chunk exceeds maximum size of {max_bytes} bytes" + ))); + } + if let Some(h) = hasher.as_mut() { + h.update(chunk); + } + if let Err(e) = file.write_all(chunk).await { + drop(file); + let _ = tokio::fs::remove_file(path).await; + return Err(AppError::internal_error(format!( + "Failed to write chunk: {e}" + ))); + } + } + } + file.flush() + .await + .map_err(|e| AppError::internal_error(format!("Failed to flush chunk file: {e}")))?; + drop(file); + + Ok(StreamedToPath { + bytes_written: total_bytes as u64, + checksum_hex: hasher.map(IncrementalHasher::finalize_hex), + alg: checksum_alg, + }) +} + +/// Algorithm-agnostic incremental hasher used by [`stream_body_to_path`] +/// and the [`ChecksumTee`] of chunked-upload completion. +/// Per-frame `update` is sub-millisecond for all three algorithms at the +/// 64 KB frame sizes axum's body stream produces, so we don't need +/// `spawn_blocking` (which the old buffered path used because it hashed +/// the full multi-MB chunk in one shot). +pub enum IncrementalHasher { + Md5(md5::Md5), + Sha256(sha2::Sha256), + // Boxing — blake3::Hasher is ~1.7 KB on the stack while md5::Md5 + // (~100 bytes) and sha2::Sha256 (~100 bytes) are tiny; boxing the + // outlier keeps the enum size proportional to the common case + // rather than the worst case. + Blake3(Box), +} + +impl IncrementalHasher { + fn new(alg: ChecksumAlg) -> Self { + match alg { + ChecksumAlg::Md5 => Self::Md5(md5::Md5::new()), + ChecksumAlg::Sha256 => Self::Sha256(sha2::Sha256::new()), + ChecksumAlg::Blake3 => Self::Blake3(Box::new(blake3::Hasher::new())), + } + } + + fn update(&mut self, bytes: &[u8]) { + match self { + Self::Md5(h) => h.update(bytes), + Self::Sha256(h) => h.update(bytes), + Self::Blake3(h) => { + h.update(bytes); + } + } + } + + fn finalize_hex(self) -> String { + match self { + Self::Md5(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Sha256(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), + Self::Blake3(h) => h.finalize().to_hex().to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + + #[tokio::test] + async fn stream_body_to_path_caps_oversized() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + // 5 MiB body, 4 MiB cap → must reject. + let body = Body::from(Bytes::from(vec![0u8; 5 * 1024 * 1024])); + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + assert!( + result.is_err(), + "expected PayloadTooLarge, got Ok(bytes_written={})", + result.ok().map(|r| r.bytes_written).unwrap_or(0) + ); + // Partial file must be removed on rejection. + assert!( + !path.exists(), + "rejected chunk file should be removed, but {} still exists", + path.display() + ); + } + + #[tokio::test] + async fn stream_body_to_path_accepts_under_cap() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 1024 * 1024])); // 1 MiB + let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; + let outcome = result.expect("should succeed"); + assert_eq!(outcome.bytes_written, 1024 * 1024); + assert!(outcome.checksum_hex.is_none(), "no alg requested → no hash"); + assert!(path.exists()); + } + + #[tokio::test] + async fn stream_body_to_path_caps_at_exact_boundary() { + // Edge case: body exactly equal to cap should succeed; cap+1 must fail. + let temp_dir = tempfile::tempdir().expect("tempdir"); + let path = temp_dir.path().join("chunk"); + + let body = Body::from(Bytes::from(vec![1u8; 100])); + let outcome = stream_body_to_path(body, &path, 100, None) + .await + .expect("100 bytes at 100-byte cap should succeed"); + assert_eq!(outcome.bytes_written, 100); + + let path2 = temp_dir.path().join("chunk2"); + let body = Body::from(Bytes::from(vec![1u8; 101])); + assert!( + stream_body_to_path(body, &path2, 100, None).await.is_err(), + "101 bytes at 100-byte cap must reject" + ); + assert!(!path2.exists()); + } + + #[tokio::test] + async fn ingest_rejects_oversized_before_touching_storage() { + // 1 KiB body against a 100-byte cap: the adapter must abort the + // stream before any flush, so the stub dedup service (which cannot + // reach PG) is never asked to settle a batch. + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from(vec![0u8; 1024]))]); + + let result = ingest_stream_to_cas( + source, + &dedup, + "file.bin", + "application/octet-stream", + 100, + None, + ) + .await; + + let err = result.err().expect("oversized body must be rejected"); + assert_eq!(err.status_code, axum::http::StatusCode::PAYLOAD_TOO_LARGE); + } + + #[tokio::test] + async fn ingest_surfaces_source_errors_as_bad_request() { + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::iter(vec![ + Ok::<_, std::io::Error>(Bytes::from_static(b"partial")), + Err(std::io::Error::other("connection reset by peer")), + ]); + + let result = + ingest_stream_to_cas(source, &dedup, "file.bin", "text/plain", usize::MAX, None).await; + + let err = result.err().expect("source error must surface"); + assert_eq!(err.status_code, axum::http::StatusCode::BAD_REQUEST); + assert!( + err.message.contains("connection reset by peer"), + "original cause must be preserved: {}", + err.message + ); + } + + /// Regression: a source shorter than the MIME peek (< MAGIC_BYTES_LEN) + /// is drained to None during sniffing and then polled once more by the + /// `chain` that re-attaches the peeked head. Non-fused sources — like + /// the `stream::unfold` used for multipart fields — panic on that + /// post-None poll ("Unfold must not be polled after it returned + /// `Poll::Ready(None)`") unless the ingest fuses the stream first. + /// The stub dedup service can't reach PG, so an orderly `Err` (not a + /// panic) proves the stream layer survived. + #[tokio::test] + async fn ingest_short_stream_with_generic_mime_does_not_repoll_source() { + let dedup = Arc::new(DedupService::new_stub()); + let source = stream::unfold(false, |done| async move { + if done { + None + } else { + Some((Ok::<_, std::io::Error>(Bytes::from_static(b"tiny")), true)) + } + }); + + let result = ingest_stream_to_cas( + source, + &dedup, + "tiny.bin", + "application/octet-stream", + usize::MAX, + None, + ) + .await; + + assert!( + result.is_err(), + "stub DB must reject the store — but only AFTER the stream \ + layer survived the post-peek poll" + ); + } + + #[tokio::test] + async fn stream_from_files_concatenates_in_order() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let a = temp_dir.path().join("a"); + let b = temp_dir.path().join("b"); + tokio::fs::write(&a, b"Hello, ").await.unwrap(); + tokio::fs::write(&b, b"World!").await.unwrap(); + + let mut out = Vec::new(); + let s = stream_from_files(vec![a, b]); + futures::pin_mut!(s); + while let Some(chunk) = s.next().await { + out.extend_from_slice(&chunk.expect("read")); + } + assert_eq!(out, b"Hello, World!"); + } + + #[tokio::test] + async fn checksum_tee_roundtrip() { + let tee = checksum_tee(ChecksumAlg::Md5); + if let Ok(mut h) = tee.lock() + && let Some(h) = h.as_mut() + { + h.update(b"hello world"); + } + let hex = finalize_checksum_tee(&tee).expect("digest"); + assert_eq!(hex, "5eb63bbbe01eeed093cb22bb8f5acdc3"); + assert!( + finalize_checksum_tee(&tee).is_none(), + "second finalize returns None" + ); + } +} diff --git a/src/interfaces/upload_spool.rs b/src/interfaces/upload_spool.rs deleted file mode 100644 index 7ac2a56a..00000000 --- a/src/interfaces/upload_spool.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! Shared streaming upload spool: request body → temp file + incremental hash. -//! -//! Used by both the native WebDAV PUT handler and the NextCloud-compat PUT -//! handler so neither buffers the full request body in memory. Peak heap is -//! ~one HTTP frame regardless of file size; the body is written to a temp -//! file (off tmpfs when [`StorageConfig::upload_temp_dir`] is configured) and -//! BLAKE3-hashed on the fly so the dedup layer can short-circuit on a hit. - -use std::path::{Path, PathBuf}; - -use axum::body::Body; -use http_body_util::BodyStream; -// The `Digest` trait (re-exported by both `md5` and `sha2` from the -// `digest` crate) gives `Md5` and `Sha256` their `new` / `update` / -// `finalize` methods. Importing once via `sha2` covers both — -// otherwise every call site would need fully-qualified -// `::…` syntax. -use sha2::Digest as _; -use tempfile::NamedTempFile; -use tokio::io::AsyncWriteExt; -use tokio_stream::StreamExt; - -use crate::application::ports::chunked_upload_ports::ChecksumAlg; -use crate::common::temp::new_spool_temp_file; -use crate::interfaces::errors::AppError; - -/// Outcome of spooling a request body to disk. -pub struct SpooledBody { - /// The temp file holding the body. Kept alive by the caller (dropping it - /// removes the file unless the dedup layer already consumed/moved it). - pub temp: NamedTempFile, - /// Hex-encoded BLAKE3 of the full body — matches `DedupService::hash_file`, - /// so passing it as `pre_computed_hash` enables the dedup fast path. - pub hash: String, - /// Total bytes written. - pub size: u64, -} - -/// Stream an HTTP request body to a temp file, computing its BLAKE3 hash -/// incrementally and enforcing `max_upload` as a hard size limit. -/// -/// Peak heap is ~one frame — the body is never fully buffered in RAM. -/// -/// `temp_dir` is taken by value (not `&Path`) so the returned future captures -/// no borrowed lifetime — required for the handler future to stay `Send`. -pub async fn spool_body_to_temp( - body: Body, - max_upload: usize, - temp_dir: Option, -) -> Result { - let temp = new_spool_temp_file(temp_dir.as_deref()) - .map_err(|e| AppError::internal_error(format!("Failed to create temp file: {e}")))?; - let temp_path = temp.path().to_path_buf(); - - let mut file = tokio::fs::File::create(&temp_path) - .await - .map_err(|e| AppError::internal_error(format!("Failed to open temp file: {e}")))?; - - let mut hasher = blake3::Hasher::new(); - let mut total_bytes: usize = 0; - let mut stream = BodyStream::new(body); - - while let Some(frame_result) = stream.next().await { - let frame = frame_result - .map_err(|e| AppError::bad_request(format!("Failed to read request body: {e}")))?; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len(); - if total_bytes > max_upload { - // Abort early — stop reading, delete temp file. - drop(file); - let _ = tokio::fs::remove_file(&temp_path).await; - return Err(AppError::payload_too_large(format!( - "Upload body exceeds the direct-PUT cap ({max_upload} bytes). \ - Use the chunked-upload protocol (REST: `/api/uploads/...`, \ - NextCloud: `/remote.php/dav/uploads/...`) for files larger than this. \ - Chunked uploads are resumable on transient failure." - ))); - } - hasher.update(chunk); - file.write_all(chunk).await.map_err(|e| { - AppError::internal_error(format!("Failed to write to temp file: {e}")) - })?; - } - } - file.flush() - .await - .map_err(|e| AppError::internal_error(format!("Failed to flush temp file: {e}")))?; - drop(file); - - let hash = hasher.finalize().to_hex().to_string(); - Ok(SpooledBody { - temp, - hash, - size: total_bytes as u64, - }) -} - -/// Result of a streamed write to a caller-supplied path. -pub struct StreamedToPath { - /// Total bytes written. - pub bytes_written: u64, - /// Lowercase hex digest, populated only when `checksum_alg=Some(_)` - /// was passed. The algorithm is identified by [`StreamedToPath::alg`]. - pub checksum_hex: Option, - /// Algorithm used to compute `checksum_hex`. Echoed back so the - /// caller can include it in audit logs or response headers. - pub alg: Option, -} - -/// Stream an HTTP request body directly to a known destination file, -/// enforcing `max_bytes` as a hard size limit. -/// -/// Used by the chunked-upload PUT handlers — each chunk has a -/// deterministic on-disk path (`NextcloudChunkedUploadService::safe_chunk_path` -/// for the NC surface, `ChunkedUploadService::prepare_chunk` for the -/// REST surface), so there's no spool/move dance. Peak heap is ~one -/// HTTP frame regardless of chunk size or `max_bytes`. -/// -/// `checksum_alg` is the optional client-requested integrity check -/// (default `md5` per the legacy `Content-MD5` contract; `blake3` -/// available for forward-compat). When `Some`, the hash is computed -/// incrementally during streaming — no extra disk read for verification. -/// -/// On size overflow the partial file is removed before the function -/// returns, so a client retry against the same chunk name starts from -/// a clean slate. On any other I/O error the partial file is also -/// removed and the error surfaces — callers can assume the path is -/// either fully written or absent. -pub async fn stream_body_to_path( - body: Body, - path: &Path, - max_bytes: usize, - checksum_alg: Option, -) -> Result { - let mut file = tokio::fs::File::create(path) - .await - .map_err(|e| AppError::internal_error(format!("Failed to open chunk file: {e}")))?; - - let mut total_bytes: usize = 0; - let mut stream = BodyStream::new(body); - let mut hasher = checksum_alg.map(IncrementalHasher::new); - - while let Some(frame_result) = stream.next().await { - let frame = match frame_result { - Ok(f) => f, - Err(e) => { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::bad_request(format!( - "Failed to read request body: {e}" - ))); - } - }; - if let Some(chunk) = frame.data_ref() { - total_bytes += chunk.len(); - if total_bytes > max_bytes { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::payload_too_large(format!( - "Chunk exceeds maximum size of {max_bytes} bytes" - ))); - } - if let Some(h) = hasher.as_mut() { - h.update(chunk); - } - if let Err(e) = file.write_all(chunk).await { - drop(file); - let _ = tokio::fs::remove_file(path).await; - return Err(AppError::internal_error(format!( - "Failed to write chunk: {e}" - ))); - } - } - } - file.flush() - .await - .map_err(|e| AppError::internal_error(format!("Failed to flush chunk file: {e}")))?; - drop(file); - - Ok(StreamedToPath { - bytes_written: total_bytes as u64, - checksum_hex: hasher.map(IncrementalHasher::finalize_hex), - alg: checksum_alg, - }) -} - -/// Algorithm-agnostic incremental hasher used by [`stream_body_to_path`]. -/// Per-frame `update` is sub-millisecond for all three algorithms at the -/// 64 KB frame sizes axum's body stream produces, so we don't need -/// `spawn_blocking` (which the old buffered path used because it hashed -/// the full multi-MB chunk in one shot). -enum IncrementalHasher { - Md5(md5::Md5), - Sha256(sha2::Sha256), - // Boxing — blake3::Hasher is ~1.7 KB on the stack while md5::Md5 - // (~100 bytes) and sha2::Sha256 (~100 bytes) are tiny; boxing the - // outlier keeps the enum size proportional to the common case - // rather than the worst case. - Blake3(Box), -} - -impl IncrementalHasher { - fn new(alg: ChecksumAlg) -> Self { - match alg { - ChecksumAlg::Md5 => Self::Md5(md5::Md5::new()), - ChecksumAlg::Sha256 => Self::Sha256(sha2::Sha256::new()), - ChecksumAlg::Blake3 => Self::Blake3(Box::new(blake3::Hasher::new())), - } - } - - fn update(&mut self, bytes: &[u8]) { - match self { - Self::Md5(h) => h.update(bytes), - Self::Sha256(h) => h.update(bytes), - Self::Blake3(h) => { - h.update(bytes); - } - } - } - - fn finalize_hex(self) -> String { - match self { - Self::Md5(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), - Self::Sha256(h) => h.finalize().iter().map(|b| format!("{b:02x}")).collect(), - Self::Blake3(h) => h.finalize().to_hex().to_string(), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use bytes::Bytes; - - #[tokio::test] - async fn stream_body_to_path_caps_oversized() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - // 5 MiB body, 4 MiB cap → must reject. - let body = Body::from(Bytes::from(vec![0u8; 5 * 1024 * 1024])); - let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; - assert!( - result.is_err(), - "expected PayloadTooLarge, got Ok(bytes_written={})", - result.ok().map(|r| r.bytes_written).unwrap_or(0) - ); - // Partial file must be removed on rejection. - assert!( - !path.exists(), - "rejected chunk file should be removed, but {} still exists", - path.display() - ); - } - - #[tokio::test] - async fn stream_body_to_path_accepts_under_cap() { - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - let body = Body::from(Bytes::from(vec![1u8; 1024 * 1024])); // 1 MiB - let result = stream_body_to_path(body, &path, 4 * 1024 * 1024, None).await; - let outcome = result.expect("should succeed"); - assert_eq!(outcome.bytes_written, 1024 * 1024); - assert!(outcome.checksum_hex.is_none(), "no alg requested → no hash"); - assert!(path.exists()); - } - - #[tokio::test] - async fn stream_body_to_path_caps_at_exact_boundary() { - // Edge case: body exactly equal to cap should succeed; cap+1 must fail. - let temp_dir = tempfile::tempdir().expect("tempdir"); - let path = temp_dir.path().join("chunk"); - - let body = Body::from(Bytes::from(vec![1u8; 100])); - let outcome = stream_body_to_path(body, &path, 100, None) - .await - .expect("100 bytes at 100-byte cap should succeed"); - assert_eq!(outcome.bytes_written, 100); - - let path2 = temp_dir.path().join("chunk2"); - let body = Body::from(Bytes::from(vec![1u8; 101])); - assert!( - stream_body_to_path(body, &path2, 100, None).await.is_err(), - "101 bytes at 100-byte cap must reject" - ); - assert!(!path2.exists()); - } -} diff --git a/static/js/core/types.js b/static/js/core/types.js index 143b75ee..594919c0 100644 --- a/static/js/core/types.js +++ b/static/js/core/types.js @@ -520,3 +520,77 @@ * @typedef {{kind: 'user', id: string} | {kind: 'group', id: string}} GroupMemberItem */ +// ------------------- Instant upload (dedup) + +/** + * Response from `GET /api/dedup/check/{hash}` — user-scoped: `exists` only + * reflects content the CALLER already owns, never global existence. + * Mirrors `HashCheckResponse` on the server (`dedup_handler.rs`). + * @typedef {Object} HashCheckAnswer + * @property {boolean} exists + * @property {string} hash BLAKE3 echoed back (64 hex chars) + * @property {number} [existing_size] size in bytes, present when `exists` + */ + +/** + * Request body for `POST /api/files/by-hash` (instant upload — registers a + * file from an already-owned blob, zero content bytes on the wire). + * Mirrors `CreateFileByHashRequest` on the server (`file_handler.rs`). + * The 201 response body is a {@link FileItem}. + * @typedef {Object} CreateFileByHash + * @property {string} name file name to create (basename only) + * @property {string} folder_id target folder (caller needs Create on it) + * @property {string} hash BLAKE3 of the owned content (64 hex chars) + */ + +// ------------------- Delta sync (chunk negotiation) + +/** + * One chunk reference on the delta wire: terse on purpose (a 10 GB file + * is ~40 000 of these). Mirrors `ChunkRef` on the server + * (`delta_upload_service.rs`). + * @typedef {Object} DeltaChunkRef + * @property {string} h BLAKE3 of the chunk (64 hex chars) + * @property {number} s chunk size in bytes (1 ..= 1 MiB) + */ + +/** + * Response of `POST /api/files/delta/negotiate` — the distinct chunk + * hashes the caller must upload (user-scoped, advisory). + * @typedef {Object} DeltaNegotiateAnswer + * @property {string[]} missing + */ + +/** + * Request body of `POST /api/files/delta/commit`. Exactly one of + * (`name` + `folder_id`) or `file_id` selects create vs update mode. + * 201/200 responses carry a {@link FileItem}; 409 carries + * `{still_missing: string[]}` (upload those chunks and retry). + * @typedef {Object} DeltaCommitRequest + * @property {string} file_hash BLAKE3 of the whole file (verified server-side) + * @property {DeltaChunkRef[]} chunks full sequence, in file order + * @property {string} [name] create mode: file name + * @property {string} [folder_id] create mode: target folder + * @property {string} [file_id] update mode: file whose content is replaced + */ + +/** + * Response of `GET /api/files/{id}/manifest` — the recipe to rebuild a + * file from chunks (delta download, step 1). Immutable per `file_hash`; + * the endpoint serves it with `ETag: file_hash` so polling sync clients + * revalidate with a 304 for free. + * @typedef {Object} DeltaManifestAnswer + * @property {string} file_hash BLAKE3 of the whole file + * @property {number} total_size bytes + * @property {DeltaChunkRef[]} chunks full sequence, in file order + */ + +/** + * Request body of `POST /api/files/delta/download` (delta download, + * step 2). Responds with `[u32 BE length][bytes]` frames in request + * order, or 404 `{not_available: string[]}` for chunks outside the + * caller's files. + * @typedef {Object} DeltaDownloadRequest + * @property {string[]} hashes distinct chunk hashes to fetch + */ + diff --git a/static/js/features/files/deltaUpload.js b/static/js/features/files/deltaUpload.js new file mode 100644 index 00000000..f6a17d14 --- /dev/null +++ b/static/js/features/files/deltaUpload.js @@ -0,0 +1,160 @@ +/** + * OxiCloud - Delta upload ("upload only what changed"). + * + * Main-thread orchestrator for `workers/deltaWorker.js`, which runs the + * whole client side of the delta protocol off the UI thread: FastCDC + * chunking + BLAKE3 (the same WASM crate and parameters as the server, + * so boundaries match bit for bit), per-batch negotiation, upload of + * only the missing chunks, and the commit. + * + * This SUBSUMES the previous whole-file instant upload: a fully known + * file negotiates to "nothing missing" and the commit short-circuits on + * possession of the file hash — same zero-byte outcome, one pipeline. + * + * Performance posture: + * - Stages overlap inside the worker (hash ‖ negotiate ‖ upload), so + * wall-clock approaches max(hash, upload) instead of their sum. + * - RAM stays flat: 8 MiB read slices; chunk bytes are re-sliced from + * the File at upload time, never hoarded. + * - Files below {@link DELTA_UPLOAD_MIN_SIZE} skip the pipeline: the + * round-trips cost more than the bytes. + * - Any failure falls back silently to the normal byte upload — delta + * is an optimization, never a gate. + */ + +import { getCsrfToken } from '../../core/csrf.js'; + +/** + * Files smaller than this upload normally: hashing + negotiation + * round-trips outweigh the transfer. + */ +export const DELTA_UPLOAD_MIN_SIZE = 8 * 1024 * 1024; + +// Absolute URL on purpose — works in dev and in the release IIFE bundle +// (same pattern as the pdf.js loader in thumbnail.js). +const DELTA_WORKER_URL = '/js/workers/deltaWorker.js'; + +/** Budget: 120 s base + 90 s per GB (hashing + uploading the delta). */ +const DELTA_TIMEOUT_BASE_MS = 120000; +const DELTA_TIMEOUT_PER_GB_MS = 90000; + +/** + * `false` once the environment proved unable to run the worker/WASM — + * later files skip straight to the byte upload. `null` = not yet known. + * @type {boolean | null} + */ +let _deltaUploadUsable = null; + +/** + * Result contract shared with the uploaders' `UploadAnswer`, plus the + * bandwidth accounting the UI surfaces. + * @typedef {Object} DeltaUploadAnswer + * @property {boolean} ok + * @property {any} [data] FileDto on success + * @property {string} [errorMsg] + * @property {boolean} [isQuotaError] + * @property {number} [savedBytes] bytes NOT transferred thanks to dedup + */ + +/** + * Try to upload `file` through the delta protocol. + * + * Resolves `null` whenever the plain byte upload should proceed (file too + * small, environment unusable, any transport/protocol failure). Resolves + * a {@link DeltaUploadAnswer} when the outcome is conclusive — success, + * quota exceeded, or a name conflict a byte upload would also hit. + * + * @param {File} file + * @param {string | null | undefined} folderId + * @param {(pct: number) => void} [onProgress] 0-99 while transferring + * @returns {Promise} + */ +export function tryDeltaUpload(file, folderId, onProgress) { + if (!folderId || file.size < DELTA_UPLOAD_MIN_SIZE || _deltaUploadUsable === false || typeof Worker === 'undefined') { + return Promise.resolve(null); + } + + return new Promise((resolve) => { + /** @type {Worker} */ + let worker; + try { + worker = new Worker(DELTA_WORKER_URL, { type: 'module' }); + } catch (_) { + _deltaUploadUsable = false; + resolve(null); + return; + } + + const sizeGB = file.size / (1024 * 1024 * 1024); + const timeoutMs = DELTA_TIMEOUT_BASE_MS + Math.ceil(sizeGB) * DELTA_TIMEOUT_PER_GB_MS; + + let savedBytes = 0; + + /** @param {DeltaUploadAnswer | null} answer */ + const settle = (answer) => { + clearTimeout(timer); + worker.terminate(); + resolve(answer); + }; + const timer = setTimeout(() => settle(null), timeoutMs); + + worker.onmessage = (event) => { + const msg = /** @type {any} */ (event.data); + if (msg.type === 'progress') { + savedBytes = msg.reusedBytes; + if (onProgress && msg.totalBytes > 0) { + const pct = Math.min(99, Math.round((100 * (msg.reusedBytes + msg.uploadedBytes)) / msg.totalBytes)); + onProgress(pct); + } + return; + } + if (msg.type === 'fallback') { + settle(null); + return; + } + if (msg.type === 'done') { + if (msg.status === 201 || msg.status === 200) { + settle({ ok: true, data: msg.body, savedBytes }); + return; + } + /** @type {string} */ + const errorMsg = msg.body?.message || msg.body?.error || `Delta upload failed (HTTP ${msg.status})`; + if (msg.status === 507) { + settle({ ok: false, isQuotaError: true, errorMsg }); + return; + } + if (msg.status === 409 && !msg.body?.still_missing) { + // Duplicate name — a byte upload would hit the same wall. + settle({ ok: false, errorMsg }); + return; + } + // still_missing exhausted, 4xx/5xx oddities: byte upload is + // the safe road (the server dedups it on write anyway). + settle(null); + } + }; + worker.onerror = () => { + // Worker script failed to load/parse — permanent environment trait. + _deltaUploadUsable = false; + settle(null); + }; + + worker.postMessage({ + file, + folderId, + name: file.name, + csrfToken: getCsrfToken() || '' + }); + }); +} + +/** + * Bilingual one-line summary for the bandwidth saved by a batch. + * @param {number} savedBytes + * @param {string} locale + * @returns {string} + */ +export function formatSavedSummary(savedBytes, locale) { + const mb = (savedBytes / (1024 * 1024)).toFixed(1); + return locale.startsWith('es') ? `Deduplicación: ${mb} MB no necesitaron subirse` : `Deduplication: ${mb} MB didn't need uploading`; +} diff --git a/static/js/features/files/fileOperations.js b/static/js/features/files/fileOperations.js index 87f7578d..f62b52d3 100644 --- a/static/js/features/files/fileOperations.js +++ b/static/js/features/files/fileOperations.js @@ -12,6 +12,7 @@ import { i18n } from '../../core/i18n.js'; import { notifications } from '../../core/notifications.js'; import { invalidateFolderMeta } from '../../model/filesModel.js'; import { triggerBrowserDownload } from '../../utils/download.js'; +import { formatSavedSummary, tryDeltaUpload } from './deltaUpload.js'; /** * @typedef {Object} BatchResult @@ -391,6 +392,7 @@ const fileOps = { let uploadedCount = 0; let successCount = 0; let quotaStop = false; + let savedBytesTotal = 0; const targetFolderId = app.currentPath || app.userHomeFolderId; @@ -404,20 +406,35 @@ const fileOps = { if (quotaStop) return; const file = readableFiles[idx]; - const formData = new FormData(); - if (targetFolderId) formData.append('folder_id', targetFolderId); - formData.append('file', file); - - console.log(`Uploading file to folder: ${targetFolderId || 'root'}`, { - file: file.name, - size: file.size + // ── Delta upload: chunk + hash locally (worker/WASM) and + // transfer only what the server doesn't already have for + // this user. Any miss/failure falls back to a byte upload. + /** @type {UploadAnswer & { savedBytes?: number } | null} */ + let result = await tryDeltaUpload(file, targetFolderId, (pct) => { + if (batchId) { + try { + notifications.updateFile(batchId, file.name, pct, 'uploading'); + } catch (_) {} + } }); + if (result) { + savedBytesTotal += result.savedBytes || 0; + if (batchId) { + try { + notifications.updateFile(batchId, file.name, 100, result.ok ? 'done' : 'error'); + } catch (_) {} + } + } else { + const formData = new FormData(); + if (targetFolderId) formData.append('folder_id', targetFolderId); + formData.append('file', file); - // Scale stall timeout with file size: - // base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit - const sizeGB = file.size / (1024 * 1024 * 1024); - const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000); - const result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout); + // Scale stall timeout with file size: + // base 120s + 60s per GB, so a 7 GB file gets ~540s stall limit + const sizeGB = file.size / (1024 * 1024 * 1024); + const dynamicTimeout = Math.max(120000, 120000 + Math.ceil(sizeGB) * 60000); + result = await this._uploadFileXHR(formData, batchId, file.name, dynamicTimeout); + } uploadedCount++; @@ -483,6 +500,14 @@ const fileOps = { // All done this._finishUploadToast(successCount, totalFiles); + if (savedBytesTotal > 0 && notifications) { + notifications.addNotification({ + icon: 'fa-bolt', + iconClass: 'upload', + title: i18n?.getCurrentLocale?.()?.startsWith('es') ? 'Subida delta' : 'Delta upload', + text: formatSavedSummary(savedBytesTotal, i18n?.getCurrentLocale?.() || 'en') + }); + } // Refresh storage usage display try { @@ -634,6 +659,7 @@ const fileOps = { let uploadedCount = 0; let successCount = 0; let quotaStop = false; + let savedBytesTotal = 0; // ── Concurrent upload with limited parallelism ────────── // FIFOs are pre-caught by the 0-byte arrayBuffer guard, @@ -663,48 +689,55 @@ const fileOps = { const parentPath = parts.slice(0, -1).join('/'); const targetFolderId = folderMap.get(parentPath) || currentFolderId; - // ── FIFO/pipe guard (0-byte files only) ── - // Named pipes (runit supervise/control) report size=0 - // but block on open(). Pre-read only 0-byte files into - // memory; files with size>0 are always regular files and - // go straight to FormData (zero extra memory copy). - /** @type {Blob} */ - let uploadFile = file; // default: use original File - if (file.size === 0) { - try { - const buf = await Promise.race([ - file.arrayBuffer(), - new Promise((_, rej) => setTimeout(() => rej(new Error('read-timeout')), 2000)) - ]); - uploadFile = new Blob([buf], { - type: file.type || 'application/octet-stream' - }); - } catch { - console.warn(`[SKIP] #${idx} ${rel} — cannot read 0-byte file (FIFO/pipe?), skipping`); - uploadedCount++; - successCount++; - if (batchId) { - try { - notifications.fileCompleted(batchId, true); - } catch (_) {} + // ── Delta upload (only changed bytes on the wire) ── + // Same fallback contract as uploadFiles: a null result + // means "do the byte upload". The shared accounting + // after this try block handles both outcomes. + const delta = await tryDeltaUpload(file, targetFolderId); + if (delta) { + result = delta; + savedBytesTotal += delta.savedBytes || 0; + } else { + // ── FIFO/pipe guard (0-byte files only) ── + // Named pipes (runit supervise/control) report size=0 + // but block on open(). Pre-read only 0-byte files into + // memory; files with size>0 are always regular files and + // go straight to FormData (zero extra memory copy). + /** @type {Blob} */ + let uploadFile = file; // default: use original File + if (file.size === 0) { + try { + const buf = await Promise.race([ + file.arrayBuffer(), + new Promise((_, rej) => setTimeout(() => rej(new Error('read-timeout')), 2000)) + ]); + uploadFile = new Blob([buf], { + type: file.type || 'application/octet-stream' + }); + } catch { + console.warn(`[SKIP] #${idx} ${rel} — cannot read 0-byte file (FIFO/pipe?), skipping`); + uploadedCount++; + successCount++; + if (batchId) { + try { + notifications.fileCompleted(batchId, true); + } catch (_) {} + } + return; } - return; } + + const formData = new FormData(); + formData.append('folder_id', targetFolderId); + formData.append('file', uploadFile, file.name); + + const thisTimeout = + file.size === 0 + ? TIMEOUT_MS_ZERO + : Math.max(TIMEOUT_MIN_MS, TIMEOUT_BASE_MS + Math.ceil(file.size / (1024 * 1024)) * TIMEOUT_PER_MB_MS); + + result = await this._uploadFileFetch(formData, thisTimeout); } - - const formData = new FormData(); - formData.append('folder_id', targetFolderId); - formData.append('file', uploadFile, file.name); - - const thisTimeout = - file.size === 0 - ? TIMEOUT_MS_ZERO - : Math.max(TIMEOUT_MIN_MS, TIMEOUT_BASE_MS + Math.ceil(file.size / (1024 * 1024)) * TIMEOUT_PER_MB_MS); - console.log(`[UPLOAD START] #${idx} ${rel} (${file.size} bytes, timeout=${thisTimeout}ms)`); - - result = await this._uploadFileFetch(formData, thisTimeout); - - console.log(`[UPLOAD END] #${idx} ${rel} ok=${result.ok}${result.errorMsg ? ` err=${result.errorMsg}` : ''}`); } catch (e) { result = { ok: false, @@ -758,6 +791,14 @@ const fileOps = { await Promise.all(workers); this._finishUploadToast(successCount, totalFiles); + if (savedBytesTotal > 0 && notifications) { + notifications.addNotification({ + icon: 'fa-bolt', + iconClass: 'upload', + title: i18n?.getCurrentLocale?.()?.startsWith('es') ? 'Subida delta' : 'Delta upload', + text: formatSavedSummary(savedBytesTotal, i18n?.getCurrentLocale?.() || 'en') + }); + } try { await refreshUserData(); diff --git a/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js b/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js new file mode 100644 index 00000000..b706d36b --- /dev/null +++ b/static/js/vendors/hash-wasm/oxicloud_hash_wasm.js @@ -0,0 +1,343 @@ +/** + * Incremental BLAKE3 hasher. + * + * ```js + * const h = new Blake3Hasher(); + * h.update(chunkBytes); // repeat per slice + * const hex = h.finalizeHex(); + * ``` + */ +export class Blake3Hasher { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + Blake3HasherFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_blake3hasher_free(ptr, 0); + } + /** + * Bytes hashed so far — lets the worker report progress without + * tracking its own counter. + * @returns {number} + */ + count() { + const ret = wasm.blake3hasher_count(this.__wbg_ptr); + return ret; + } + /** + * Finish and return the lowercase hex digest (64 chars). The hasher + * can keep receiving `update` calls afterwards (BLAKE3 finalization + * is non-destructive), but the frontend treats it as terminal. + * @returns {string} + */ + finalizeHex() { + let deferred1_0; + let deferred1_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.blake3hasher_finalizeHex(retptr, this.__wbg_ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred1_0 = r0; + deferred1_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred1_0, deferred1_1, 1); + } + } + /** + * Create a fresh hasher. + */ + constructor() { + const ret = wasm.blake3hasher_new(); + this.__wbg_ptr = ret; + Blake3HasherFinalization.register(this, this.__wbg_ptr, this); + return this; + } + /** + * Feed one slice of the file. + * @param {Uint8Array} data + */ + update(data) { + const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.blake3hasher_update(this.__wbg_ptr, ptr0, len0); + } +} +if (Symbol.dispose) Blake3Hasher.prototype[Symbol.dispose] = Blake3Hasher.prototype.free; + +/** + * Incremental FastCDC chunker + whole-file BLAKE3, for the delta-upload + * worker. Feed the file in slices; every call returns the chunks that + * became FINAL; `finish()` flushes the tail and returns the file hash. + * + * ```js + * const c = new DeltaChunker(); + * for (const slice of slices) { + * for (const [h, s] of JSON.parse(c.update(bytes))) { … } + * } + * const { chunks, file_hash } = JSON.parse(c.finish()); + * ``` + * + * Correctness of the incremental split: FastCDC decides each cut by + * scanning at most `CDC_MAX_CHUNK` bytes from the chunk's start. When + * the chunker runs over the buffered prefix of a longer file, every + * produced chunk except the LAST ended on a content/max-size condition + * — its decision window was fully available, so the full-file chunker + * makes the same cut. Only the last chunk (cut by "end of buffer") is + * provisional: it stays buffered and is re-examined when more bytes + * arrive. By induction the emitted boundaries equal a single FastCDC + * pass over the whole file — the mirror test below proves it. + */ +export class DeltaChunker { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + DeltaChunkerFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_deltachunker_free(ptr, 0); + } + /** + * Flush the provisional tail and return + * `{"chunks":[["",size]…],"file_hash":"","total":N}`. + * `chunks` holds at most one entry (the tail); an empty file has none + * and its `file_hash` is BLAKE3 of the empty input. + * @returns {string} + */ + finish() { + let deferred1_0; + let deferred1_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + wasm.deltachunker_finish(retptr, this.__wbg_ptr); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred1_0 = r0; + deferred1_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred1_0, deferred1_1, 1); + } + } + /** + * Create a chunker with the server's CDC parameters. + */ + constructor() { + const ret = wasm.deltachunker_new(); + this.__wbg_ptr = ret; + DeltaChunkerFinalization.register(this, this.__wbg_ptr, this); + return this; + } + /** + * Feed one slice. Returns a JSON array of the chunks that became + * final: `[["", size], …]` (possibly empty). + * @param {Uint8Array} data + * @returns {string} + */ + update(data) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.deltachunker_update(retptr, this.__wbg_ptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred2_0, deferred2_1, 1); + } + } +} +if (Symbol.dispose) DeltaChunker.prototype[Symbol.dispose] = DeltaChunker.prototype.free; + +/** + * One-shot convenience for small buffers. + * @param {Uint8Array} data + * @returns {string} + */ +export function blake3Hex(data) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray8ToWasm0(data, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.blake3Hex(retptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export2(deferred2_0, deferred2_1, 1); + } +} +function __wbg_get_imports() { + const import0 = { + __proto__: null, + __wbg___wbindgen_throw_bbadd78c1bac3a77: function(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); + }, + }; + return { + __proto__: null, + "./oxicloud_hash_wasm_bg.js": import0, + }; +} + +const Blake3HasherFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_blake3hasher_free(ptr, 1)); +const DeltaChunkerFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_deltachunker_free(ptr, 1)); + +let cachedDataViewMemory0 = null; +function getDataViewMemory0() { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; +} + +function getStringFromWasm0(ptr, len) { + return decodeText(ptr >>> 0, len); +} + +let cachedUint8ArrayMemory0 = null; +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +function passArray8ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 1, 1) >>> 0; + getUint8ArrayMemory0().set(arg, ptr / 1); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); +cachedTextDecoder.decode(); +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +let WASM_VECTOR_LEN = 0; + +let wasmModule, wasmInstance, wasm; +function __wbg_finalize_init(instance, module) { + wasmInstance = instance; + wasm = instance.exports; + wasmModule = module; + cachedDataViewMemory0 = null; + cachedUint8ArrayMemory0 = null; + return wasm; +} + +async function __wbg_load(module, imports) { + if (typeof Response === 'function' && module instanceof Response) { + if (typeof WebAssembly.instantiateStreaming === 'function') { + try { + return await WebAssembly.instantiateStreaming(module, imports); + } catch (e) { + const validResponse = module.ok && expectedResponseType(module.type); + + if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { + console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); + + } else { throw e; } + } + } + + const bytes = await module.arrayBuffer(); + return await WebAssembly.instantiate(bytes, imports); + } else { + const instance = await WebAssembly.instantiate(module, imports); + + if (instance instanceof WebAssembly.Instance) { + return { instance, module }; + } else { + return instance; + } + } + + function expectedResponseType(type) { + switch (type) { + case 'basic': case 'cors': case 'default': return true; + } + return false; + } +} + +function initSync(module) { + if (wasm !== undefined) return wasm; + + + if (module !== undefined) { + if (Object.getPrototypeOf(module) === Object.prototype) { + ({module} = module) + } else { + console.warn('using deprecated parameters for `initSync()`; pass a single object instead') + } + } + + const imports = __wbg_get_imports(); + if (!(module instanceof WebAssembly.Module)) { + module = new WebAssembly.Module(module); + } + const instance = new WebAssembly.Instance(module, imports); + return __wbg_finalize_init(instance, module); +} + +async function __wbg_init(module_or_path) { + if (wasm !== undefined) return wasm; + + + if (module_or_path !== undefined) { + if (Object.getPrototypeOf(module_or_path) === Object.prototype) { + ({module_or_path} = module_or_path) + } else { + console.warn('using deprecated parameters for the initialization function; pass a single object instead') + } + } + + if (module_or_path === undefined) { + module_or_path = new URL('oxicloud_hash_wasm_bg.wasm', import.meta.url); + } + const imports = __wbg_get_imports(); + + if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { + module_or_path = fetch(module_or_path); + } + + const { instance, module } = await __wbg_load(await module_or_path, imports); + + return __wbg_finalize_init(instance, module); +} + +export { initSync, __wbg_init as default }; diff --git a/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm b/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm new file mode 100644 index 00000000..0f09d2a6 Binary files /dev/null and b/static/js/vendors/hash-wasm/oxicloud_hash_wasm_bg.wasm differ diff --git a/static/js/workers/deltaWorker.js b/static/js/workers/deltaWorker.js new file mode 100644 index 00000000..3a86f650 --- /dev/null +++ b/static/js/workers/deltaWorker.js @@ -0,0 +1,330 @@ +/** + * OxiCloud — delta-upload worker ("upload only what changed"). + * + * Runs the whole client side of the delta protocol off the main thread: + * + * read 8 MiB slices ─► FastCDC chunk + BLAKE3 (WASM, same crate and + * parameters as the server) ─► negotiate hash batches ─► upload only + * the missing chunks (framed, bounded concurrency) ─► commit. + * + * The stages OVERLAP: negotiation of batch N and uploads of its missing + * chunks run while batch N+1 is still being hashed, so wall-clock time + * approaches max(hash time, upload time) instead of their sum. RAM stays + * flat: chunk bytes are re-sliced from the File at upload time, never + * hoarded. + * + * Protocol with the spawner: + * in : { file: File, folderId: string, name: string, csrfToken: string } + * out : { type: 'progress', hashedBytes, reusedBytes, uploadedBytes, totalBytes } + * { type: 'done', status, body } — conclusive HTTP outcome + * { type: 'fallback', reason } — do a plain byte upload + */ + +// Absolute URLs on purpose: vendors/workers are served verbatim in both +// dev and the release IIFE bundle (same pattern as the pdf.js loader). +const WASM_GLUE_URL = '/js/vendors/hash-wasm/oxicloud_hash_wasm.js'; + +/** File read granularity — large enough to amortize Blob→ArrayBuffer. */ +const SLICE_BYTES = 8 * 1024 * 1024; +/** Negotiate after this many freshly hashed chunks (~64 MiB of content). */ +const NEGOTIATE_BATCH = 256; +/** Group missing chunks into PUT bodies of at most this many bytes. */ +const UPLOAD_BATCH_BYTES = 8 * 1024 * 1024; +/** Concurrent chunk-PUT requests. */ +const UPLOAD_CONCURRENCY = 2; +/** Re-commit attempts when the server answers 409 still_missing. */ +const COMMIT_RETRIES = 2; + +/** + * Typed view of the dedicated-worker global scope (jsconfig targets the + * DOM lib, where `self` is a Window — cast to what this worker uses). + * @type {{ onmessage: ((event: MessageEvent) => void) | null, + * postMessage: (message: unknown) => void }} + */ +const workerScope = /** @type {any} */ (self); + +/** + * One chunk occurrence, in file order. + * @typedef {{ h: string, s: number, offset: number }} WorkerChunk + */ + +/** @returns {Promise} the initialized WASM module */ +async function loadWasm() { + const mod = await import(WASM_GLUE_URL); + await mod.default(); + return mod; +} + +workerScope.onmessage = async (event) => { + const { file, folderId, name, csrfToken } = /** @type {{ file: File, folderId: string, name: string, csrfToken: string }} */ (event.data); + + /** @param {string} reason */ + const fallback = (reason) => workerScope.postMessage({ type: 'fallback', reason }); + + /** @type {Record} */ + const mutHeaders = { 'Content-Type': 'application/json' }; + if (csrfToken) mutHeaders['X-CSRF-Token'] = csrfToken; + + let wasm; + try { + wasm = await loadWasm(); + } catch (err) { + fallback(`wasm unavailable: ${err instanceof Error ? err.message : String(err)}`); + return; + } + + // ── Shared pipeline state ───────────────────────────────────── + /** @type {WorkerChunk[]} */ + const chunks = []; // every occurrence, in file order + /** @type {Set} */ + const seenForNegotiate = new Set(); // distinct hashes already sent to negotiate + let reusedBytes = 0; + let uploadedBytes = 0; + let hashedBytes = 0; + let failed = /** @type {string | null} */ (null); + + let lastProgress = 0; + const progress = (force = false) => { + const now = Date.now(); + if (!force && now - lastProgress < 150) return; + lastProgress = now; + workerScope.postMessage({ + type: 'progress', + hashedBytes, + reusedBytes, + uploadedBytes, + totalBytes: file.size + }); + }; + + // ── Upload stage: bounded-concurrency drain of uploadByHash ── + /** @type {WorkerChunk[]} */ + const uploadQueue = []; + /** @type {Promise[]} */ + const uploadWorkers = []; + let uploadsClosed = false; + /** @type {(() => void) | null} */ + let wakeUploader = null; + const signalUploaders = () => { + if (wakeUploader) { + const w = wakeUploader; + wakeUploader = null; + w(); + } + }; + + /** Encode a batch of chunks as [u32 BE len][bytes] frames. */ + const encodeFrames = async (/** @type {WorkerChunk[]} */ batch) => { + const total = batch.reduce((n, c) => n + 4 + c.s, 0); + const wire = new Uint8Array(total); + const view = new DataView(wire.buffer); + let at = 0; + for (const c of batch) { + // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM + const bytes = new Uint8Array(await file.slice(c.offset, c.offset + c.s).arrayBuffer()); + view.setUint32(at, c.s, false); + wire.set(bytes, at + 4); + at += 4 + c.s; + } + return wire; + }; + + const uploadLoop = async () => { + while (!failed) { + // Take up to UPLOAD_BATCH_BYTES from the queue. + /** @type {WorkerChunk[]} */ + const batch = []; + let bytes = 0; + while (uploadQueue.length > 0 && bytes < UPLOAD_BATCH_BYTES) { + const c = /** @type {WorkerChunk} */ (uploadQueue.shift()); + batch.push(c); + bytes += c.s; + } + if (batch.length === 0) { + if (uploadsClosed) return; + // eslint-disable-next-line no-await-in-loop -- queue wait + await new Promise((resolve) => { + wakeUploader = /** @type {() => void} */ (resolve); + }); + continue; + } + try { + // eslint-disable-next-line no-await-in-loop -- bounded by pool size + const wire = await encodeFrames(batch); + // eslint-disable-next-line no-await-in-loop -- bounded by pool size + const response = await fetch('/api/files/delta/chunks', { + method: 'PUT', + headers: { + 'Content-Type': 'application/octet-stream', + ...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {}) + }, + body: wire + }); + if (!response.ok) { + failed = `chunk PUT failed (HTTP ${response.status})`; + return; + } + for (const c of batch) uploadedBytes += c.s; + progress(); + } catch (err) { + failed = `chunk PUT failed: ${err instanceof Error ? err.message : String(err)}`; + return; + } + } + }; + for (let i = 0; i < UPLOAD_CONCURRENCY; i++) uploadWorkers.push(uploadLoop()); + + // ── Negotiate stage ─────────────────────────────────────────── + /** @type {Promise[]} */ + const negotiations = []; + const negotiate = (/** @type {WorkerChunk[]} */ fresh) => { + if (fresh.length === 0 || failed) return; + negotiations.push( + (async () => { + try { + const response = await fetch('/api/files/delta/negotiate', { + method: 'POST', + headers: mutHeaders, + body: JSON.stringify({ chunks: fresh.map(({ h, s }) => ({ h, s })) }) + }); + if (!response.ok) { + failed = failed || `negotiate failed (HTTP ${response.status})`; + return; + } + const missing = new Set(/** @type {{missing: string[]}} */ (await response.json()).missing); + for (const c of fresh) { + if (missing.has(c.h)) { + uploadQueue.push(c); + } else { + reusedBytes += c.s; + } + } + signalUploaders(); + progress(); + } catch (err) { + failed = failed || `negotiate failed: ${err instanceof Error ? err.message : String(err)}`; + } + })() + ); + }; + + // ── Chunking stage (drives the other two) ──────────────────── + try { + const chunker = new wasm.DeltaChunker(); + /** @type {WorkerChunk[]} */ + let freshBatch = []; + let offset = 0; + + /** @param {[string, number][]} emitted */ + const onChunks = (emitted) => { + for (const [h, s] of emitted) { + /** @type {WorkerChunk} */ + const chunk = { h, s, offset }; + offset += s; + chunks.push(chunk); + if (seenForNegotiate.has(h)) { + // Repeated content inside the same file: the first + // occurrence decides upload vs reuse; later ones are + // pure reuse for accounting. + reusedBytes += s; + } else { + seenForNegotiate.add(h); + freshBatch.push(chunk); + if (freshBatch.length >= NEGOTIATE_BATCH) { + negotiate(freshBatch); + freshBatch = []; + } + } + } + }; + + for (let read = 0; read < file.size && !failed; read += SLICE_BYTES) { + const end = Math.min(read + SLICE_BYTES, file.size); + // eslint-disable-next-line no-await-in-loop -- sequential by design: constant RAM + const slice = new Uint8Array(await file.slice(read, end).arrayBuffer()); + onChunks(JSON.parse(chunker.update(slice))); + hashedBytes = end; + progress(); + } + const fin = JSON.parse(chunker.finish()); + chunker.free(); + onChunks(fin.chunks); + negotiate(freshBatch); + const fileHash = /** @type {string} */ (fin.file_hash); + hashedBytes = file.size; + progress(true); + + // ── Drain: negotiations → uploads → commit ─────────────── + await Promise.all(negotiations); + uploadsClosed = true; + signalUploaders(); + await Promise.all(uploadWorkers); + if (failed) { + fallback(failed); + return; + } + + const commitBody = { + file_hash: fileHash, + chunks: chunks.map(({ h, s }) => ({ h, s })), + name, + folder_id: folderId + }; + for (let attempt = 0; ; attempt++) { + // eslint-disable-next-line no-await-in-loop -- retry loop + const response = await fetch('/api/files/delta/commit', { + method: 'POST', + headers: mutHeaders, + body: JSON.stringify(commitBody) + }); + /** @type {any} */ + let body = null; + try { + // eslint-disable-next-line no-await-in-loop -- retry loop + body = await response.json(); + } catch (_) {} + + const stillMissing = response.status === 409 && Array.isArray(body?.still_missing); + if (stillMissing && attempt < COMMIT_RETRIES) { + // GC race or a chunk we wrongly assumed claimable: upload + // exactly what the server names and try again. + const byHash = new Map(chunks.map((c) => [c.h, c])); + /** @type {WorkerChunk[]} */ + const retry = []; + for (const h of body.still_missing) { + const c = byHash.get(h); + if (!c) { + fallback('server requested an unknown chunk'); + return; + } + retry.push(c); + } + const wire = await encodeFrames(retry); + // eslint-disable-next-line no-await-in-loop -- retry loop + const put = await fetch('/api/files/delta/chunks', { + method: 'PUT', + headers: { + 'Content-Type': 'application/octet-stream', + ...(csrfToken ? { 'X-CSRF-Token': csrfToken } : {}) + }, + body: wire + }); + if (!put.ok) { + fallback(`retry chunk PUT failed (HTTP ${put.status})`); + return; + } + for (const c of retry) uploadedBytes += c.s; + progress(true); + continue; + } + + // Conclusive: 201 created, or a real error (quota, name + // conflict, validation). The spawner maps it to the uploaders' + // UploadAnswer contract. + workerScope.postMessage({ type: 'done', status: response.status, body }); + return; + } + } catch (err) { + fallback(err instanceof Error ? err.message : String(err)); + } +}; diff --git a/wasm/oxicloud-hash/Cargo.lock b/wasm/oxicloud-hash/Cargo.lock new file mode 100644 index 00000000..5675d509 --- /dev/null +++ b/wasm/oxicloud-hash/Cargo.lock @@ -0,0 +1,191 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "fastcdc" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af40d8a8dadb92dc178569a5f5edb5f3056e98255c2de48ab5d59a52892e0c" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oxicloud-hash-wasm" +version = "0.1.0" +dependencies = [ + "blake3", + "fastcdc", + "wasm-bindgen", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasm-bindgen" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +dependencies = [ + "unicode-ident", +] diff --git a/wasm/oxicloud-hash/Cargo.toml b/wasm/oxicloud-hash/Cargo.toml new file mode 100644 index 00000000..b95fd253 --- /dev/null +++ b/wasm/oxicloud-hash/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "oxicloud-hash-wasm" +version = "0.1.0" +edition = "2021" +description = "BLAKE3 hashing for the OxiCloud web frontend — compiled from the exact same crate the server uses, so client-side hashes match server-side content addressing bit for bit." +publish = false + +# Standalone workspace root: this crate is built only by +# scripts/build-wasm.sh (wasm32 target) and must not join the server +# workspace — `cargo test --workspace` / clippy on the host have nothing +# useful to do with a wasm cdylib. +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +# Same major as the server's Cargo.toml. `wasm32_simd` enables the WASM +# SIMD128 kernels (~3-4× over the portable implementation); every +# evergreen browser since 2023 supports it, and the frontend falls back +# to a plain byte upload when instantiation fails. +blake3 = { version = "1.8.4", default-features = false, features = ["wasm32_simd"] } +# Same crate AND parameters as the server's CDC dedup engine — chunk +# boundaries computed in the browser must equal the server's bit for bit, +# or cross-version dedup between byte uploads and delta uploads collapses. +fastcdc = "4.0.0" +wasm-bindgen = "0.2" + +[profile.release] +# Hashing throughput is the whole point of this module. +opt-level = 3 +lto = "fat" +codegen-units = 1 +strip = true diff --git a/wasm/oxicloud-hash/src/lib.rs b/wasm/oxicloud-hash/src/lib.rs new file mode 100644 index 00000000..627c379b --- /dev/null +++ b/wasm/oxicloud-hash/src/lib.rs @@ -0,0 +1,319 @@ +//! BLAKE3 for the OxiCloud web frontend. +//! +//! Compiled from the same `blake3` crate the server uses, so a hash +//! computed in the browser equals the server's content address bit for +//! bit — the property the instant-upload path depends on. +//! +//! The API is incremental on purpose: the worker feeds the file in +//! slices (`Blob.slice().arrayBuffer()`), keeping RAM constant no matter +//! how large the file is. + +use wasm_bindgen::prelude::*; + +/// Incremental BLAKE3 hasher. +/// +/// ```js +/// const h = new Blake3Hasher(); +/// h.update(chunkBytes); // repeat per slice +/// const hex = h.finalizeHex(); +/// ``` +#[wasm_bindgen] +pub struct Blake3Hasher { + inner: blake3::Hasher, +} + +#[wasm_bindgen] +impl Blake3Hasher { + /// Create a fresh hasher. + #[wasm_bindgen(constructor)] + pub fn new() -> Blake3Hasher { + Blake3Hasher { + inner: blake3::Hasher::new(), + } + } + + /// Feed one slice of the file. + pub fn update(&mut self, data: &[u8]) { + self.inner.update(data); + } + + /// Finish and return the lowercase hex digest (64 chars). The hasher + /// can keep receiving `update` calls afterwards (BLAKE3 finalization + /// is non-destructive), but the frontend treats it as terminal. + #[wasm_bindgen(js_name = finalizeHex)] + pub fn finalize_hex(&self) -> String { + self.inner.finalize().to_hex().to_string() + } + + /// Bytes hashed so far — lets the worker report progress without + /// tracking its own counter. + pub fn count(&self) -> f64 { + self.inner.count() as f64 + } +} + +impl Default for Blake3Hasher { + fn default() -> Self { + Self::new() + } +} + +/// One-shot convenience for small buffers. +#[wasm_bindgen(js_name = blake3Hex)] +pub fn blake3_hex(data: &[u8]) -> String { + blake3::hash(data).to_hex().to_string() +} + +// ── Delta-upload chunker ───────────────────────────────────────────────────── + +/// CDC parameters — MUST mirror `dedup_service.rs` on the server +/// (`CDC_MIN_CHUNK` / `CDC_AVG_CHUNK` / `CDC_MAX_CHUNK`). Identical +/// parameters + identical crate ⇒ identical boundaries, which is what +/// makes a chunk hashed in the browser deduplicate against a chunk the +/// server cut from a byte upload. +const CDC_MIN_CHUNK: usize = 65_536; +const CDC_AVG_CHUNK: usize = 262_144; +const CDC_MAX_CHUNK: usize = 1_048_576; + +/// Incremental FastCDC chunker + whole-file BLAKE3, for the delta-upload +/// worker. Feed the file in slices; every call returns the chunks that +/// became FINAL; `finish()` flushes the tail and returns the file hash. +/// +/// ```js +/// const c = new DeltaChunker(); +/// for (const slice of slices) { +/// for (const [h, s] of JSON.parse(c.update(bytes))) { … } +/// } +/// const { chunks, file_hash } = JSON.parse(c.finish()); +/// ``` +/// +/// Correctness of the incremental split: FastCDC decides each cut by +/// scanning at most `CDC_MAX_CHUNK` bytes from the chunk's start. When +/// the chunker runs over the buffered prefix of a longer file, every +/// produced chunk except the LAST ended on a content/max-size condition +/// — its decision window was fully available, so the full-file chunker +/// makes the same cut. Only the last chunk (cut by "end of buffer") is +/// provisional: it stays buffered and is re-examined when more bytes +/// arrive. By induction the emitted boundaries equal a single FastCDC +/// pass over the whole file — the mirror test below proves it. +#[wasm_bindgen] +pub struct DeltaChunker { + /// Provisional tail: bytes after the last FINAL cut. + buf: Vec, + file_hasher: blake3::Hasher, + total: u64, +} + +/// Append one `["",len]` item to a hand-rolled JSON array — hashes +/// are hex and sizes are integers, so manual JSON is unambiguous and +/// keeps a serde dependency out of the wasm binary. +fn push_chunk_json(out: &mut String, hash: &str, len: usize) { + if !out.ends_with('[') { + out.push(','); + } + out.push_str("[\""); + out.push_str(hash); + out.push_str("\","); + out.push_str(&len.to_string()); + out.push(']'); +} + +#[wasm_bindgen] +impl DeltaChunker { + /// Create a chunker with the server's CDC parameters. + #[wasm_bindgen(constructor)] + pub fn new() -> DeltaChunker { + DeltaChunker { + buf: Vec::with_capacity(2 * CDC_MAX_CHUNK), + file_hasher: blake3::Hasher::new(), + total: 0, + } + } + + /// Feed one slice. Returns a JSON array of the chunks that became + /// final: `[["", size], …]` (possibly empty). + pub fn update(&mut self, data: &[u8]) -> String { + self.file_hasher.update(data); + self.total += data.len() as u64; + self.buf.extend_from_slice(data); + + let mut out = String::from("["); + let mut consumed = 0usize; + { + let chunks: Vec = fastcdc::v2020::FastCDC::new( + &self.buf, + CDC_MIN_CHUNK, + CDC_AVG_CHUNK, + CDC_MAX_CHUNK, + ) + .collect(); + // Every chunk but the last ended on a content/max condition → + // final. The last one ended because the buffer did → keep it. + for chunk in chunks.iter().take(chunks.len().saturating_sub(1)) { + let bytes = &self.buf[chunk.offset..chunk.offset + chunk.length]; + push_chunk_json( + &mut out, + &blake3::hash(bytes).to_hex().to_string(), + chunk.length, + ); + consumed = chunk.offset + chunk.length; + } + } + if consumed > 0 { + self.buf.drain(..consumed); + } + out.push(']'); + out + } + + /// Flush the provisional tail and return + /// `{"chunks":[["",size]…],"file_hash":"","total":N}`. + /// `chunks` holds at most one entry (the tail); an empty file has none + /// and its `file_hash` is BLAKE3 of the empty input. + pub fn finish(&mut self) -> String { + let mut out = String::from("{\"chunks\":["); + if !self.buf.is_empty() { + push_chunk_json( + &mut out, + &blake3::hash(&self.buf).to_hex().to_string(), + self.buf.len(), + ); + self.buf.clear(); + } + out.push_str("],\"file_hash\":\""); + out.push_str(&self.file_hasher.finalize().to_hex().to_string()); + out.push_str("\",\"total\":"); + out.push_str(&self.total.to_string()); + out.push('}'); + out + } +} + +impl Default for DeltaChunker { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The vector the frontend smoke test uses — also proves the wasm + /// build hashes identically to the server (same crate, same output). + #[test] + fn hello_world_vector() { + let hasher = { + let mut h = Blake3Hasher::new(); + h.update(b"Hello, "); + h.update(b"World!"); + h + }; + assert_eq!( + hasher.finalize_hex(), + "288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8" + ); + assert_eq!( + blake3_hex(b"Hello, World!"), + "288a86a79f20a3d6dccdca7713beaed178798296bdfa7913fa2a62d9727bf8f8" + ); + } + + #[test] + fn empty_input_vector() { + assert_eq!( + blake3_hex(b""), + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" + ); + } + + // ── DeltaChunker mirror test ───────────────────────────────── + // + // The client-side twin of the server's + // `test_stream_chunking_matches_slice_chunking`: incremental chunking + // with adversarial slice sizes must produce exactly the boundaries of + // one FastCDC pass over the whole buffer — the property cross-version + // dedup between byte uploads and delta uploads hangs on. + + fn run_chunker(data: &[u8], slice: usize) -> (Vec<(String, usize)>, String) { + let mut chunker = DeltaChunker::new(); + let mut chunks: Vec<(String, usize)> = Vec::new(); + let mut parse = |json: &str, into: &mut Vec<(String, usize)>| { + // items look like ["",N] — split on '[' groups. + for item in json.split("[\"").skip(1) { + let hash = &item[..64]; + let size: usize = item[66..item.find(']').unwrap()].parse().unwrap(); + into.push((hash.to_string(), size)); + } + }; + for piece in data.chunks(slice.max(1)) { + let emitted = chunker.update(piece); + parse(&emitted, &mut chunks); + } + let fin = chunker.finish(); + let tail_json = &fin[fin.find('[').unwrap()..=fin.find(']').unwrap()]; + parse(tail_json, &mut chunks); + let file_hash = fin.split("\"file_hash\":\"").nth(1).unwrap()[..64].to_string(); + (chunks, file_hash) + } + + #[test] + fn incremental_chunking_matches_single_pass() { + // 4 MiB of xorshift noise — genuinely content-defined cut points + // (a byte-periodic generator would only ever hit max-size cuts). + let mut state: u64 = 0x243F_6A88_85A3_08D3; + let mut data = Vec::with_capacity(4 * 1024 * 1024); + while data.len() < 4 * 1024 * 1024 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + data.extend_from_slice(&state.to_le_bytes()); + } + + let reference: Vec<(String, usize)> = + fastcdc::v2020::FastCDC::new(&data, CDC_MIN_CHUNK, CDC_AVG_CHUNK, CDC_MAX_CHUNK) + .map(|c| { + ( + blake3::hash(&data[c.offset..c.offset + c.length]) + .to_hex() + .to_string(), + c.length, + ) + }) + .collect(); + assert!(reference.len() > 4, "test data must span several chunks"); + + // Slice sizes chosen to stress every refill path: tiny (7 B), + // typical worker slice (8 MiB > file), page-ish, and exactly the + // CDC max so provisional tails land on boundaries. + for slice in [7usize, 4096, CDC_MAX_CHUNK, 8 * 1024 * 1024] { + let (chunks, file_hash) = run_chunker(&data, slice); + assert_eq!( + chunks, reference, + "boundaries must not depend on slicing (slice={slice})" + ); + assert_eq!( + file_hash, + blake3_hex(&data), + "file hash must match one-shot BLAKE3 (slice={slice})" + ); + } + } + + #[test] + fn delta_chunker_empty_and_tiny_inputs() { + let (chunks, file_hash) = run_chunker(b"", 1024); + assert!(chunks.is_empty()); + assert_eq!( + file_hash, + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" + ); + + let tiny = b"below the CDC minimum"; + let (chunks, file_hash) = run_chunker(tiny, 4); + assert_eq!(chunks.len(), 1, "tiny input is one (tail) chunk"); + assert_eq!(chunks[0].1, tiny.len()); + assert_eq!(chunks[0].0, blake3_hex(tiny)); + assert_eq!(file_hash, blake3_hex(tiny)); + } +}