From 4cb73eaf3982349e9ed90728d65f62b4052de252 Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sat, 1 Aug 2026 20:42:24 +0200 Subject: [PATCH 01/29] plan(storage-key-rotation): add a key rotation + header version blob --- docs/plan/storage-key-rotation.md | 609 ++++++++++++++++++++++++++++++ 1 file changed, 609 insertions(+) create mode 100644 docs/plan/storage-key-rotation.md diff --git a/docs/plan/storage-key-rotation.md b/docs/plan/storage-key-rotation.md new file mode 100644 index 00000000..b75098b2 --- /dev/null +++ b/docs/plan/storage-key-rotation.md @@ -0,0 +1,609 @@ +# Plan — Storage encryption-key rotation + +## Context + +Today an encrypted storage entry declares a single key: + +```env +OXICLOUD_STORAGE_s3_prod_BACKEND=s3 +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY= +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_CIPHER=aes-256-gcm # optional, defaults to aes-256-gcm +``` + +Blobs land at `.blob` as raw AES-GCM output (`nonce | ciphertext | tag`) — +no discriminating prefix. That key can never change. Any real deployment needs +to rotate keys — after a suspected leak, on a periodic policy, or after a staff +turnover. The current answer is "create a second entry and run a migration to +it," which: + +* forces the admin to provision a second bucket / directory, +* churns object storage costs, +* and does not scale to routine rotations. + +The proper fix is **in-place key rotation** inside the same entry. The storage +config holds a *list* of keys, the server writes with the newest one, reads +with any of them, and a background job re-encrypts every blob under the newest +key. When the job completes, the admin removes the old key from the list. + +At the same time we solve two related on-disk problems that today's format +leaves open: + +1. **No self-description.** A `.blob` today is ambiguous: could be raw + plaintext (unencrypted deployment) or raw AES-GCM output (encrypted + deployment). Reads guess based on the entry config; a misconfigured key or + a wrong-cipher default silently returns garbage. +2. **No path from encrypted to plaintext.** Once encryption is on for an + entry, the only way off is via a second entry and full migration — same + ergonomics problem as key rotation. + +Both fall out of introducing a self-describing on-blob header (`v1`). The +object-key suffix stays `.blob` for both eras — the magic bytes at the +top of every v1 blob are the discriminator, so legacy files stay readable +byte-identically and new files coexist alongside them at the same suffix. +Migration is lazy. + +Related memory (this plan supersedes it): + +> **[Encryption key rotation in place — deferred, unsafe today]** — the previous +> "namespace object keys by encryption generation" direction (`.k2.blob`) +> is DROPPED in favour of the pair-list + v1-header approach below. That +> approach fought content-addressability by generation-namespacing per key; +> this one keeps a stable content-addressable object key and encodes generation +> in a compact on-blob header. + +## Design + +### The pair-list config + +Replace the singular `_ENCRYPTION_KEY` + `_ENCRYPTION_CIPHER` with a single +comma-separated list of pairs on `_ENCRYPTION_KEY` alone: + +```env +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: +``` + +Rules: + +* **Format** — `[:]` per pair, comma-separated. Whitespace + around commas / colons tolerated. +* **Cipher optional.** Only one real cipher (`aes-256-gcm`) exists today, so + `` on its own is legal and behaves as `aes-256-gcm:`. When a + second cipher lands, the colon-prefixed form is the disambiguator. +* **The `none` cipher.** Legal, has no key material (`none:` — trailing colon + with an empty key). Enables migrating BOTH ways: encrypt a previously-plaintext + deployment (`none:,aes-256-gcm:`), or decrypt an encrypted deployment + (`aes-256-gcm:,none:`). Correctness relies on the v1 header (see + *Encrypted-blob format*) discriminating encrypted vs plaintext, NOT on + fallback ordering. A `none` pair by itself in the list is equivalent to + omitting `_ENCRYPTION_KEY` entirely (kept for symmetry). +* **Order matters. Last pair wins on writes.** For v1-header reads, the + `key_fp` in the header selects the exact pair; the list order is irrelevant + at read time. For legacy reads (see *Coexistence*), the head pair is used + because pre-v1 deployments had only one key by construction. +* **At least one pair.** Empty list → boot aborts. +* **Uniqueness.** Same key appearing twice = boot aborts (config drift smell). + `none` may appear at most once. +* **Fingerprint.** For each real-cipher pair, log a truncated SHA-256 of the + key material at boot (12 hex chars) so a swap-order accident is loud in the + audit stream. `none` logs as `none:—`. + +### Retire `_ENCRYPTION_CIPHER` + +`_ENCRYPTION_CIPHER` was added recently and has not shipped in a release. Drop +it entirely in the same slice — the pair-list makes it redundant, and the on-blob +v1 header carries the cipher choice explicitly per blob anyway. No back-compat +shim needed. + +### Encrypted-blob format (v1 header) + +Every v1 blob starts with a fixed self-describing header. Layout: + +``` +"OXCPT" 5 bytes — magic marker (0x4F 0x58 0x43 0x50 0x54) + 2 bytes — big-endian u16; v1 = 0x0001 + 8 bytes — sha256(key material)[..8]; identifies which pair + decrypts. For a `none` (plaintext) v1 blob this is + eight zero bytes and the fields below are absent. + 12 bytes — random per blob (AES-GCM standard nonce length) + N bytes — encrypted payload; same length as the plaintext + 16 bytes — AEAD authentication tag + ──── +fixed overhead: 43 bytes for encrypted v1; 15 bytes for plaintext v1 +``` + +* **The magic `OXCPT`** unambiguously distinguishes a v1 blob from legacy raw + bytes. It's the sole discriminator — no filename convention, no DB column, + the first five bytes tell the reader everything. +* **``** is the only field that can change format without touching + every blob. v1 is what this plan ships; v2 slots are reserved + (see *Forward compatibility*). +* **``** eliminates guess-work on read. The pair-list lookup becomes + O(1) — no fallback tag-check loop. A key-fp with no matching pair means + `NoKeyForBlob` (500 with a clear message), NEVER silent garbage. +* **``** and **``** are the AES-GCM primitive's own outputs. + Their length is fixed by v1 = AES-256-GCM. If a future v2 uses a different + AEAD with the same shape (say ChaCha20-Poly1305, tag = 16 B, nonce = 12 B) + v2 can keep the layout; a truly-different AEAD triggers a new version. +* **Plaintext v1 blobs** (produced when the entry's head pair is `none`) use + the same magic + version + all-zero key_fp, then the raw plaintext bytes + directly (no nonce, no ciphertext framing, no tag). The header alone is + enough to say "this is plaintext v1", cleanly distinguished from encrypted + v1 and from legacy raw-plaintext. + +**Why no CRC.** The AEAD tag covers ciphertext + nonce integrity. Content +addressability (`` = BLAKE3(plaintext)) covers whole-blob integrity for +either format. A CRC over the header adds nothing — a corrupt version byte +fails on decode; a corrupt key_fp fails to find a pair. Both fail loud. + +### Coexistence with legacy blobs + +Legacy and v1 blobs share the `.blob` object-key namespace. The magic +bytes at position 0 discriminate them safely: + +| First 5 bytes | Interpretation | Read path | +|---|---|---| +| `"OXCPT"` | v1 blob | verify magic, dispatch on version + key_fp | +| anything else | legacy | apply entry config: raw plaintext if no encryption pair, single-key AES-GCM decode with the head pair if encryption is set (by construction pre-v1 used one key, and any pair-list produced by upgrading from a pre-v1 config keeps that key at the head) | + +Collision probability of legacy bytes accidentally matching `"OXCPT"` at +position 0 is 2⁻⁴⁰ per blob. If a match does happen, the read continues into +the v1 path, fails at version/key_fp check with `UnsupportedBlobVersion` or +`NoKeyForBlob`, and returns a hard error. **Never silent corruption.** For +context: on a 10 M-blob deployment, expected number of collision hard-errors +across the lifetime of legacy files is ~10⁻⁵ blobs. Effectively never. + +Properties this buys us: + +* **No forced migration.** Legacy `.blob` files stay readable + indefinitely. Existing deployments upgrade to v1 code with zero + format-conversion work. +* **Lazy conversion on hot paths.** Any COW overwrite (WebDAV MOVE, PUT-over, + content-hash re-upload) naturally lands as v1 at the same object key. +* **Explicit conversion via `storage_rotate`.** The rotate job walks + `storage.blobs`, reads each blob via the magic-byte dispatch, and if the + blob is not already v1 with the head-pair key, PUTs it back as v1 with + the head pair — in place, same object key. +* **No schema changes.** Nothing new in `storage.blobs`; nothing for + `blobs_consistency`, `storage_migrate`, or any other sibling job to learn. + +### Legacy-pair guardrail + +Once any legacy blob remains on disk, the pre-v1 single key that produced +those blobs MUST remain in the pair-list at the head position (or in the list +somewhere the legacy read path can find it). Removing it early breaks every +legacy read. + +Guardrail: + +* Boot logs a `warn` line if any pair-list has more than one entry AND the + head pair's fingerprint differs from the second entry's — signals "you + added new keys but haven't rotated legacy blobs yet". +* A **legacy-blob counter** is surfaced in the admin panel per storage entry. + The counter is maintained by `blobs_consistency`: during its normal walk it + branches on the magic-byte check and records the legacy count as a run + statistic on `jobs.recoverable_runs` (existing surface, no schema hit). The + admin panel reads the most recent count and displays it. Refresh cadence is + whatever consistency scan cadence the deployment has (weekly by default; on + demand from the admin panel). +* The *"Rotation complete — safe to remove the old key"* hint appears in the + entry card only when the last `storage_rotate` run completed with zero + findings AND the most recent consistency scan reported zero legacy blobs. +* Nothing enforces removal at code level — the admin is trusted, given a + clear signal, and warned. + +### Read path + +Per blob read: + +1. Fetch `.blob` from the backend. +2. Check first 5 bytes. +3. If `"OXCPT"` → v1 read path: + * Read ``. Not `0x0001` → `UnsupportedBlobVersion`. + * Read ``. If zero → plaintext-v1; return raw bytes (post-header + payload). Else find the pair whose `sha256(key)[..8]` matches; not + found → `NoKeyForBlob`; found → AES-GCM decrypt with `` / + `` / `` and return plaintext. +4. Otherwise → legacy read path: + * If entry has any real-cipher pair, attempt AES-GCM decrypt with the + head pair's key; return plaintext on success, `DecryptFailed` on tag + failure. + * If entry has only `none` (or no pair at all), return raw bytes as + plaintext. + +Never falls through silently. Every failure is a distinct typed error the +handler can map to a 500 with an actionable message. + +### Write path + +1. Always writes to `.blob`. +2. Head pair is `none` → write v1 header (magic + version + zero key_fp) + + raw plaintext. +3. Head pair is a real cipher → compute AEAD; write v1 header (magic + + version + key_fp) + nonce + ciphertext + auth_tag. + +v1 code never produces a legacy-format blob. Any leftover legacy blobs on +disk pre-date the v1-code deployment. + +### The rotation job + +New `RecoverableJobHandler` tenant, `storage_rotate`. Mirrors +`storage_migration`'s shape: + +* **Iterates `storage.blobs`** in hash-lex order. Cursor is the last-processed + hash (64 hex chars). Same cursor encoding as `storage_migration` and + `blobs_consistency`. +* **Per blob:** + 1. Fetch `.blob` and dispatch via the standard read path. + 2. Decide if a rewrite is needed based on what actually decoded: + * Legacy blob (no v1 magic) → always rewrite (upgrade to v1 with head + pair). + * v1 encrypted, decrypted under a pair-index other than head → rewrite + (key rotation). + * v1 encrypted, already under head — skip. + * v1 plaintext, head pair is `none` — skip. + * v1 plaintext, head pair is a real cipher — rewrite + (encrypt-in-place upgrade). + * v1 encrypted, head pair is `none` — rewrite + (decrypt-in-place downgrade). + 3. Write via the standard v1 write path — same object key, atomic + replace. + 4. Checkpoint. On per-blob failure, record a `rotation_failed` finding + with severity `data_loss` (bytes may not have crossed), continue. +* **Concurrency-safe by construction.** + * A concurrent upload during rotation writes v1 with the head key. The + rotate walk will short-circuit on that hash if it reaches it later. + * A COW overwrite is the same story. + * The v1 write is atomic at object-storage level (S3 replace, Local + rename-into-place). A concurrent reader sees either state. + * No readonly mode. This is a critical improvement over + `storage_migration`: rotation is per-blob idempotent, so we don't need + to freeze writes. +* **Restart-survivable** — same boot-time sweep as every other recoverable + handler. +* **`?deep=true` unused** — rotation has no "slow variant" mode. Parameter + accepted for uniformity, ignored. + +### Admin UX + +In the admin storage-panel entry card, add a **Rotate encryption key** action. +Preconditions: + +* Entry has encryption enabled with ≥ 2 pairs OR entry has legacy blobs + outstanding OR the pair-list has otherwise changed since the last rotation. +* Otherwise the button is disabled with the tooltip *"Add a second pair in + `OXICLOUD_STORAGE__ENCRYPTION_KEY` first, or wait for legacy blobs to + accumulate — nothing to rotate right now."* + +Clicking the button dispatches the `storage_rotate` job for that entry. The +job's progress rides on the same `X-Server-Status` header infrastructure the +maintenance banner uses — but this time WITHOUT engaging read-only mode. +Banner variant reads *"Rotating encryption key on `` — X% (Y / Z +blobs). All operations continue normally."* and disappears on completion. + +The entry card shows a **legacy-blob counter** sourced from the most recent +`blobs_consistency` run: *"N legacy-format blobs remain (upgrade included in +next rotation)"*. Refresh-on-demand button next to it triggers a targeted +`blobs_consistency` scan (already available via the admin surface). The +*"Rotation complete — safe to remove the old key"* hint appears only when +N = 0 and the last `storage_rotate` run completed with zero findings. + +### Removing the old pair + +Not automated. The admin edits `.env`, restarts. Rationale: + +* We do not want the running server to silently mutate its own `.env`. +* A retained old pair is a benign cost (nothing hits it on read since + `key_fp` lookup is O(1)). +* Explicit human step matches the "add a new pair → restart" symmetry. + +## Deployment flow + +### First-time upgrade to v1 + +Zero admin work required. On upgrade: + +* v1 code deploys. +* New blobs are written in v1 format at the existing `.blob` object + key. +* Existing legacy blobs stay readable via the magic-byte dispatch — the + legacy read path is preserved verbatim. +* Admin can optionally trigger a `storage_rotate` run to consolidate every + legacy blob into v1 format. Not required — legacy blobs migrate + opportunistically via COW overwrites and stay readable indefinitely + otherwise. + +### Rotating an encryption key + +The user-facing recipe (goes verbatim into `docs/guide/backend-storage.md`): + +``` +1. Generate a new key: + openssl rand -base64 32 + +2. Append it to the entry's key list. Order matters — the NEW key goes LAST: + OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: + +3. Restart OxiCloud. New uploads are now encrypted with the new key; existing + blobs still decrypt with the old one. + +4. In the admin panel, click "Rotate encryption key" on the entry. This + dispatches a background job that re-encrypts every existing blob under + the new key AND upgrades any remaining legacy-format blobs to v1. All + operations keep working during rotation. + +5. Wait for the job to complete (progress shows in the top banner and in + the Jobs admin page). + +6. Remove the OLD key from the list: + OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm: + +7. Restart OxiCloud. Rotation is complete. +``` + +### Encrypting a previously-plaintext deployment + +``` +1. Generate a new key (as above). +2. Add it AFTER `none`: + OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY=none:,aes-256-gcm: +3. Restart. New uploads are encrypted; existing plaintext blobs stay readable. +4. Run `storage_rotate` to encrypt existing blobs in place. +5. Remove `none:` from the list; restart. +``` + +### Decrypting an encrypted deployment + +``` +1. Add `none:` AFTER the current encryption key: + OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY=aes-256-gcm:,none: +2. Restart. New uploads are plaintext; existing encrypted blobs stay readable. +3. Run `storage_rotate` to decrypt existing blobs in place. +4. Remove the key pair, keep `none:` only (or drop `_ENCRYPTION_KEY` entirely); + restart. +``` + +## Testing strategy + +**Most of this ships as Rust tests, not Hurl.** The rotation surface is +byte-level, cryptographic, and stateful across restart boundaries — none of +which Hurl can meaningfully assert. Push almost everything down into +`#[cfg(test)]` blocks alongside the code, following the codebase's existing +inline-tests convention (see AGENTS.md — "tests are primarily `#[cfg(test)]` +modules within source files"). + +**What lives in Rust (unit + integration):** + +* **Pair-list parser.** Unit tests in `config.rs` on `_ENCRYPTION_KEY` parsing: + 1-pair, 2-pair, cipher-optional shape, `none:` alone, `none:,aes:K`, + `aes:K,none:`, whitespace tolerance, duplicate rejection, multiple-`none` + rejection, empty-list rejection, retired-`_ENCRYPTION_CIPHER` rejection, + base64 length validation. Table-driven; fast. +* **v1 header round-trip.** Unit tests in + `encrypted_blob_backend.rs::tests`: encrypt payload → verify header bytes + (magic, version, key_fp) → decrypt → assert plaintext identity. One test per + format flavour (encrypted, plaintext-with-`none`-head). Verify `` = + BLAKE3(plaintext) with no offset for plaintext-v1 (the "recovery tool" + invariant). +* **Legacy read-path preservation.** Fixture: a hand-crafted raw AES-GCM + ciphertext (`nonce | ct | tag`) produced by the pre-v1 code path. Assert + that magic-byte dispatch falls through to the legacy branch and decrypts + correctly. Fixture stays in-tree as `tests/fixtures/legacy_blob.bin` so + regressions on the legacy path can never sneak through. +* **`OXCPT` collision on random data.** Property test: generate random N-byte + payloads, feed through legacy read path. Assert the ~2⁻⁴⁰ magic-collision + case fails with `UnsupportedBlobVersion` or `NoKeyForBlob` — hard error, + never silent misread. Keeps the "collisions can't silently corrupt" claim + in the plan honest. +* **Rotation decision tree.** Unit tests on the per-blob `decide()` helper of + `storage_rotate_service.rs`. All six cases from *The rotation job* section + as separate tests with clear names (`legacy_upgrades_to_v1`, + `v1_encrypted_under_head_skips`, + `v1_encrypted_under_older_pair_rewrites`, + `v1_plaintext_with_none_head_skips`, + `v1_plaintext_encrypts_when_head_is_cipher`, + `v1_encrypted_decrypts_when_head_is_none`). +* **Recoverable-job round-trip.** Integration test in + `storage_rotate_service::tests` using the existing recoverable-run harness: + seed N blobs (mix of legacy + v1-under-old-key), trigger rotation, assert + every blob ends v1-with-head, `format_generation` (if we add it later) or + the consistency-scan count reports zero legacy remaining, findings=0. +* **Crash recovery.** Same harness: interrupt mid-run at cursor position K, + restart, assert resume from K and eventual completion with correct final + state. Same discipline `storage_migration` already uses. +* **Concurrency safety.** Test that a `put_blob` call during a rotation + targeting the same hash produces exactly one v1 blob at end-state (either + the rotate's or the concurrent write's — both are head-format so the final + state is indistinguishable). Verifies the "per-blob idempotent" claim. +* **Config-restart semantics.** Table-driven test on the pair-list state + machine: start with pair-list [K1], add K2 → [K1,K2], run rotation, + drop K1 → [K2], assert every blob still decrypts. Exercises the "safe to + remove old key" transition end-to-end without going through the admin UI. + +**What Hurl covers (thin — the API surface, not the mechanics):** + +* Trigger endpoint auth: `POST /api/admin/storage/entries/{name}/rotate` is + admin-only (403 for non-admins, 401 for unauthenticated). +* Trigger endpoint preconditions: 400 or similar when pairs < 2 AND no + legacy blobs exist AND pair-list hasn't changed. +* Trigger endpoint concurrency: second POST while a run is Active returns + 409. +* Progress header: `X-Server-Status` includes a `rotation` payload during a + running job and drops it on completion. + +Hurl does NOT try to: +* Inspect on-disk bytes. +* Restart the server with a new pair-list. +* Seed a legacy blob directly. +* Verify decryption after old-key removal. + +Those all belong in Rust tests where we can hold the pool, the backend, and +the config in the same test's memory. + +**Test data.** Legacy-blob fixtures are byte-frozen (`tests/fixtures/*.bin`) +and committed. Don't regenerate them from live code — the whole point is that +they were produced by pre-v1 code and won't ever again be. Regeneration +scripts (kept outside `tests/`) are OK for one-time refreshes if the format +changes. + +## Slices + +Same discipline as `storage-multi-entry.md`. Each slice is a mergeable +increment; no slice depends on a later one. + +### Slice K1 — Pair-list config parsing + +**Scope.** Config layer only. No behaviour change on the write / read path +yet — the parser produces `Vec` and the existing code keeps calling +`pairs.last()`. + +* `NamedStorageEntry.encryption` becomes `Option>` where + `KeyPair = { cipher: CipherKind, key_material: Option<[u8; 32]> }`. + `CipherKind` = `{ AesGcm256, None }`; `None` carries no key. +* Parser: split on `,`, per-pair split on `:` (1 or 2 parts), decode base64 + when a real cipher, reject empty list, reject duplicates, reject multiple + `none`. +* Retire `OXICLOUD_STORAGE__ENCRYPTION_CIPHER` — parser errors on it with + guidance to move the cipher into the pair. +* Boot logs each pair's fingerprint (`sha256(key)[..12]` for real ciphers, + `—` for `none`) and marks the head. Warn line if head fp differs from any + non-head fp (rotation window signal). +* Docs: `example.env`, `docs/config/env.md`, `docs/guide/backend-storage.md` + updated with the pair syntax and the three recipes (rotate / encrypt / + decrypt). + +**Exit criteria.** Boot with a 1-pair config behaves identically to today. +Boot with a 2-pair config succeeds and logs both fingerprints. Boot with +`_ENCRYPTION_CIPHER` alongside `_ENCRYPTION_KEY` fails with a migration hint. + +### Slice K2 — v1 read/write paths in `EncryptedBlobBackend` + +**Scope.** The encrypted-backend decorator learns the v1 header format. Reads +dispatch on magic bytes; writes always emit v1 headers. Legacy read path +preserved verbatim. `blobs_consistency` gains a magic-byte branch to track +the legacy-blob count. + +* `EncryptedBlobBackend` refactored around `BlobFormat::V1` writer + reader. + Legacy path preserved but read-only from new code. +* Read dispatches on magic; write always emits v1 header at `.blob`. +* `blobs_consistency`: during its normal walk, per-blob magic-byte check; + legacy count recorded on the run's `stats` JSON. No schema hit — reuses + the existing `stats` bag on `jobs.recoverable_runs`. +* `storage_migrate`: no changes needed. It copies raw bytes between backends; + format is preserved on the target automatically because the object bytes + are opaque to it. + +**Exit criteria.** A brand-new deployment writes only v1 blobs. An upgraded +deployment reads existing legacy blobs and writes new v1 blobs at the same +object-key. `blobs_consistency` reports a legacy-blob count in its run stats. + +### Slice K3 — The `storage_rotate` recoverable job + +**Scope.** New handler, admin-triggered, iterates blobs, per-blob decision +tree (legacy → v1 upgrade, v1 with old key → v1 with head key, plaintext ↔ +encrypted where applicable), records findings. + +* New file `src/infrastructure/services/storage_rotate_service.rs`. +* Registered in `JobRegistry` as `storage_rotate`. Runs on the same + recoverable-runs engine (crash recovery, cursor persistence, pause/resume). +* Trigger endpoint: `POST /api/admin/storage/entries/{name}/rotate`. + Requires admin. Refuses if no work would happen (all blobs already at + head format + head key). Refuses if a `storage_rotate` or + `storage_migration` run is already Active for any entry. +* Per-blob decision tree per *The rotation job* section above. In-place + atomic replace at the same `.blob` object key. +* No readonly mode engaged. `X-Server-Status` header payload gains a + `rotation` field alongside `migration`. + +**Exit criteria.** On a filled sandbox: (1) legacy-only entry rotates to +v1; (2) encrypted entry with 2 pairs rotates so head-pair-only remains +decryptable; (3) plaintext entry with `none,cipher` head rotates to +encrypted-v1; (4) encrypted entry with `cipher,none` head rotates to +plaintext-v1. Each round-trip verified by dropping the retired pair from +config and successfully reading every blob. + +### Slice K4 — Admin panel action + banner + docs + +**Scope.** UI wiring, banner variant, guide docs, legacy-blob counter. + +* Entry card: **Rotate encryption key** button + tooltip states as above. +* Legacy-blob counter chip on each entry card. Sources the count from the + most recent `blobs_consistency` run's stats (already surfaced in the Jobs + admin page). Refresh button triggers a fresh scan. +* `ReadOnlyBanner.svelte` gains a third variant `variant="rotating"` — same + shape, milder tone (info not warning), copy makes it clear that operations + continue. +* Server-status header payload gains `rotation?: {entry, migrated, total, + percent}` alongside the existing `migration?:` field. `readonly` stays + false during rotation. +* `docs/guide/backend-storage.md`: new **Rotating an encryption key**, + **Encrypting a plaintext deployment**, and **Decrypting an encrypted + deployment** sections with the recipes above. +* `docs/config/env.md`: pair syntax section under `_ENCRYPTION_KEY`. +* Admin panel *"Rotation complete — safe to remove the old key"* hint gated + on findings=0 AND legacy count=0. + +**Exit criteria.** Round-trip on a real deployment (Ed's OVH S3): add pair +→ restart → rotate → verify blobs (including legacy) → remove old pair → +restart → blobs still readable. + +## Forward compatibility (v2 and beyond) + +The `` field is the single load-bearing knob for future format +changes. Reserved slots: + +* `0x0001` — this plan. AES-256-GCM, server-side keys, individual blob + integrity via AEAD tag + content-hash. +* `0x0002…0x00FF` — reserved for server-side format bumps: new AEAD, + chain-authenticated CDC chunks (either *manifest HMAC* — no header changes, + `chunk_manifests` gains a server-computed HMAC over the ordered chunk-hash + list — or *Merkle root in header* — v2 grows to carry `manifest_root` + + `chunk_index`, letting any streamed chunk be verified in isolation). Both + defend against manifest reorder / truncation / injection attacks that + content-addressability alone doesn't cover. +* `0x0100…0x01FF` — reserved for client-side encryption (E2E) variants. + `` becomes the client-key fingerprint; the server can no longer + decrypt and passes ciphertext through the streaming path unchanged. + +v1 and v2 coexist in the same storage indefinitely — the magic-byte read +dispatch handles arbitrary versions at position 5-6. Migration between +generations reuses `storage_rotate`'s pattern: rewrite each blob with the +new-generation writer, in-place at the same object key. + +## Non-goals + +* **Asymmetric / KMS-backed keys.** Out of scope. Symmetric AES-256-GCM only. + A KMS-backed variant is a separate slice built on top of this one. +* **Chunk-chain integrity / manifest tampering defense.** Recognised as a + real gap (chunk reorder / truncate / inject via DB write). Deferred to the + v2 header slice — see *Forward compatibility*. +* **Client-side encryption (E2E).** Same — v2 slot reserved, separate slice. +* **Automatic old-key removal.** Explicit human step, see above. +* **Rotation of a key on the "in-transit" side** (client → server TLS). + Handled by the reverse proxy; unrelated. +* **Per-blob key derivation** (KDF from a master key + blob hash). A + legitimate hardening path but orthogonal to the rotation story; folds in as + a future slice under this same pair-list config. +* **Rollback safety across format generations.** If v1 code is rolled back + to pre-v1 after new v1 blobs have been written, old code will 500 on those + blobs (AES-GCM tag fails on OXCPT-prefixed bytes). Not addressed by this + plan; the safety net is restore-from-backup. Deploying an + incompatibility-breaking format change is a pre-planned event, not a hot + rollback scenario. + +## Open questions + +* **Should we throttle the rotate job?** Same question `storage_migration` had. + Answer: not in v1. If throughput bites, add a `_ROTATE_MAX_MB_PER_SEC` on + the entry later. +* **Should rotation be idempotent under repeat trigger?** Yes. Running it a + second time with the same config is a no-op walk (every blob is already at + head format + head key). +* **Should we ship a `format_generation` column on `storage.blobs` as a + query-optimizer?** No in v1. Consistency-scan stats cover the "how many + legacy blobs remain" question at admin cadence, and per-read magic-byte + dispatch is a 5-byte compare. If a real hot-path or admin-panel latency + need emerges, the column is a small back-fill migration to add later. +* **Should the legacy-blob counter block key removal at code level?** No — + keeping the trust model consistent with "admin edits .env, we don't fight + them" is the current default. The counter + hint is enough. If real + incidents happen we can escalate to a hard guard later. From 03c8f87f1f98a748f6bd0629b9fa44afcf9bfcdf Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sat, 1 Aug 2026 21:04:58 +0200 Subject: [PATCH 02/29] feat(storage key rot): prepare format :,:,... --- src/common/config.rs | 479 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 479 insertions(+) diff --git a/src/common/config.rs b/src/common/config.rs index 16992c0d..ee53b08b 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -422,6 +422,279 @@ impl EncryptionCipher { } } +// ───────────────────────────────────────────────────────────────────── +// K1: pair-list encryption config (post-storage-multi-entry, pre-v1-header). +// See `docs/plan/storage-key-rotation.md`. +// +// The types and parser below REPLACE the singular `EncryptionCipher` + +// `encryption_key_base64` + `encryption_cipher` model with a +// list-of-pairs model where the last pair wins on writes and every +// pair is a candidate for reads. The `none` cipher is a first-class +// citizen so plaintext ↔ encrypted transitions can be expressed as a +// single pair-list evolution. +// +// K1 introduces these types but does NOT yet wire them into +// `NamedStorageEntry` — that flip lands in K2 alongside the v1 header +// read/write paths. The parser is exercised via unit tests only in +// this slice. +// ───────────────────────────────────────────────────────────────────── + +/// The AEAD (or absence of one) used by a single [`KeyPair`]. +/// +/// Distinct from the older [`EncryptionCipher`] enum in two ways: +/// +/// * Adds a `None` variant. A pair with `CipherKind::None` says +/// "writes routed to this pair produce raw plaintext". This is +/// what makes the encrypt-a-plaintext-deployment and +/// decrypt-an-encrypted-deployment recipes expressible without a +/// second storage entry — see `docs/plan/storage-key-rotation.md` +/// §"Encrypting a previously-plaintext deployment" and the +/// symmetric decrypt recipe. +/// * Is the type carried by the v1 on-disk header's cipher field +/// (indirectly via `` in v1, but explicitly if a future +/// v2 adds a cipher byte to the header — the enum grows without +/// touching call sites). +/// +/// Kept alongside [`EncryptionCipher`] during K1; K2 retires the +/// older enum in the same slice that flips `NamedStorageEntry`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CipherKind { + /// AES-256 in Galois/Counter Mode. 96-bit nonce + 128-bit tag. + /// The only real AEAD OxiCloud ships today. On the wire (v1 + /// header): `[12-byte nonce] [ciphertext] [16-byte tag]`. + AesGcm256, + /// No cipher. Writes produce raw plaintext; reads return raw + /// bytes. Used as the head pair for a plaintext-target rotation + /// (`aes:K,none:`) or as a non-head legacy pair while an + /// encrypt-in-place rotation is still upgrading old plaintext + /// blobs (`none:,aes:K`). At most one `none` pair may appear in + /// a list (parser-enforced). + None, +} + +impl CipherKind { + /// Parse an env-var token: `"aes-256-gcm"` (with the + /// `aes256gcm` alias kept for continuity with + /// [`EncryptionCipher::parse`]) or `"none"`. Case-insensitive. + pub fn parse(raw: &str) -> Option { + match raw.to_ascii_lowercase().as_str() { + "aes-256-gcm" | "aes256gcm" => Some(CipherKind::AesGcm256), + "none" => Some(CipherKind::None), + _ => None, + } + } + + /// Stable env-var-friendly name — the exact string the parser + /// accepts back and the string admin surfaces render. + pub fn as_str(self) -> &'static str { + match self { + CipherKind::AesGcm256 => "aes-256-gcm", + CipherKind::None => "none", + } + } + + /// `true` iff this cipher carries key material. `false` for + /// `CipherKind::None`. Used by the parser to enforce + /// "no key after `none:`" and by K2's write path to skip the + /// AEAD call. + pub fn needs_key(self) -> bool { + matches!(self, CipherKind::AesGcm256) + } +} + +/// One `:` pair from an `_ENCRYPTION_KEY` list. +/// +/// List order carries semantics — the LAST pair is the write pair +/// (see `docs/plan/storage-key-rotation.md` §"The pair-list config"). +/// `key_material` is `Some(bytes)` when `cipher.needs_key()`, else +/// `None`. Base64 is decoded once at parse time; downstream code +/// takes the raw bytes directly (no re-decoding on every read). +/// +/// Deliberately not `Copy` — a 32-byte key isn't cheap enough to +/// silently `Copy` and cloning it in tests is a good deterrent +/// against accidental leaks into logs. +#[derive(Debug, Clone)] +pub struct KeyPair { + /// Which AEAD (or none) this pair writes with. + pub cipher: CipherKind, + /// Raw 32-byte AES-256 key. Always `Some` for real ciphers, + /// always `None` for `CipherKind::None`. This invariant is + /// enforced at parse time; downstream can `unwrap()` when + /// `cipher.needs_key()` returns `true`. + pub key_material: Option<[u8; 32]>, +} + +impl KeyPair { + /// Truncated SHA-256 fingerprint of the key material — 12 hex + /// chars (6 bytes of SHA output). Used at boot for the audit- + /// line dump so operators can eyeball which key is at each + /// position without seeing the raw material. + /// + /// The on-blob v1 header uses a DIFFERENT truncation — 8 bytes + /// / 16 hex chars — so this fingerprint is not usable as the + /// header's `` field. Kept short here to keep boot + /// logs tight. + /// + /// Returns `None` for `CipherKind::None` (nothing to + /// fingerprint) — callers render as `—` in that case. + pub fn fingerprint_short(&self) -> Option { + use sha2::{Digest, Sha256}; + let mat = self.key_material.as_ref()?; + let full = Sha256::digest(mat); + Some(hex::encode(&full[..6])) + } +} + +/// Parse the `OXICLOUD_STORAGE__ENCRYPTION_KEY` env var value +/// into an ordered pair list. +/// +/// Grammar (informal): +/// +/// ```text +/// pair_list := pair ("," pair)* +/// pair := (cipher ":")? material +/// cipher := "aes-256-gcm" | "none" (case-insensitive) +/// material := base64_key (for real ciphers) +/// | ε (for `none:`) +/// ``` +/// +/// * Whitespace around commas / colons is tolerated. +/// * A pair without a colon defaults its cipher to `aes-256-gcm` +/// (since that's the only shipping AEAD today; new ciphers +/// MUST use the explicit `:` form). +/// * A `none` pair MUST use the explicit `none:` form (with the +/// trailing colon and empty material) — omitting the colon +/// would be ambiguous with a real key that happens to base64 +/// to `none`. +/// +/// Guaranteed non-empty on `Ok`. All error messages carry the +/// entry name so operators see which env var failed. +/// +/// Errors: +/// * Empty list. +/// * Empty pair (leading / trailing / duplicate comma). +/// * Unknown cipher name. +/// * `none` pair with non-empty material. +/// * More than one `none` pair. +/// * Real-cipher pair with empty material. +/// * Non-base64 key material. +/// * Wrong-length key material (≠ 32 bytes). +/// * Duplicate key material (same 32 bytes twice). +pub fn parse_encryption_pair_list(entry_name: &str, raw: &str) -> Result, String> { + use base64::Engine; + let raw = raw.trim(); + if raw.is_empty() { + return Err(format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY is empty — set at least \ + one `:` pair, or omit the variable entirely for an \ + unencrypted entry." + )); + } + + let mut pairs: Vec = Vec::new(); + let mut seen_none = false; + let mut seen_keys: Vec<[u8; 32]> = Vec::new(); + + for (idx0, pair_raw) in raw.split(',').enumerate() { + let pos = idx0 + 1; + let pair_raw = pair_raw.trim(); + if pair_raw.is_empty() { + return Err(format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY has an empty pair at \ + position {pos} — remove leading/trailing/duplicate commas." + )); + } + + // `split_once(':')` gives us (cipher, key); no colon = key-only, + // implicit AES-256-GCM (the only shipping real cipher). + let (cipher_tok, key_b64) = match pair_raw.split_once(':') { + Some((c, k)) => (c.trim(), k.trim()), + None => ("aes-256-gcm", pair_raw), + }; + + let cipher = CipherKind::parse(cipher_tok).ok_or_else(|| { + format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY pair {pos} has unknown \ + cipher `{cipher_tok}` — supported: `aes-256-gcm`, `none`." + ) + })?; + + if !cipher.needs_key() { + if !key_b64.is_empty() { + return Err(format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY pair {pos} declares \ + cipher `none` but has key material — use `none:` (trailing \ + colon, empty key)." + )); + } + if seen_none { + return Err(format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY has more than one \ + `none` pair — at most one is allowed." + )); + } + seen_none = true; + pairs.push(KeyPair { + cipher, + key_material: None, + }); + continue; + } + + // Real cipher — decode + length-check the key. + if key_b64.is_empty() { + return Err(format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY pair {pos} has empty \ + key material for cipher `{}`.", + cipher.as_str() + )); + } + let decoded = base64::engine::general_purpose::STANDARD + .decode(key_b64) + .map_err(|e| { + format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY pair {pos} is not \ + valid base64: {e}" + ) + })?; + if decoded.len() != 32 { + return Err(format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY pair {pos} decodes to \ + {} bytes; must be exactly 32 bytes (AES-256).", + decoded.len() + )); + } + let mut key = [0u8; 32]; + key.copy_from_slice(&decoded); + + if seen_keys.iter().any(|k| k == &key) { + return Err(format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY has the same key \ + material twice — each pair must be unique." + )); + } + seen_keys.push(key); + + pairs.push(KeyPair { + cipher, + key_material: Some(key), + }); + } + + // Belt-and-braces: the loop above rejects empty pairs, so this + // can only fire if the whole raw input was pure whitespace, which + // we already caught at the top. Kept as an invariant guard so a + // future refactor can't silently produce an empty vec. + if pairs.is_empty() { + return Err(format!( + "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY produced no pairs after \ + parsing — this should never happen; please file a bug." + )); + } + + Ok(pairs) +} + /// One named storage entry declared in `.env`. /// /// See `docs/plan/storage-multi-entry.md`. Each entry is a fully-realised @@ -3357,4 +3630,210 @@ mod tests { self.name == other.name && self.backend == other.backend } } + + // ───────────────────────────────────────────────────────────── + // K1: pair-list encryption parser tests. + // + // Pure function; no env-var seeding needed. Table-driven where + // the shape allows, individual tests where the error message is + // load-bearing. + // ───────────────────────────────────────────────────────────── + mod pair_list_parser { + use super::*; + + /// 32-byte base64 string, deterministic across tests. Two + /// distinct valid keys for multi-pair tests. + const K1_B64: &str = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; // 0x00..0x1F + const K2_B64: &str = "IHwgP2AVE1E6VwlbT8BjSggJc9OjNXJDKf8bF19HYPU="; // random + + #[test] + fn single_key_no_cipher_prefix_defaults_to_aes_gcm() { + let pairs = parse_encryption_pair_list("t", K1_B64).unwrap(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].cipher, CipherKind::AesGcm256); + assert!(pairs[0].key_material.is_some()); + } + + #[test] + fn single_key_with_explicit_cipher_prefix() { + let pairs = parse_encryption_pair_list("t", &format!("aes-256-gcm:{K1_B64}")).unwrap(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].cipher, CipherKind::AesGcm256); + } + + #[test] + fn cipher_prefix_is_case_insensitive() { + for tok in [ + "aes-256-gcm", + "AES-256-GCM", + "Aes-256-Gcm", + "aes256gcm", + "AES256GCM", + ] { + let pairs = parse_encryption_pair_list("t", &format!("{tok}:{K1_B64}")).unwrap(); + assert_eq!(pairs.len(), 1, "failed on token `{tok}`"); + assert_eq!(pairs[0].cipher, CipherKind::AesGcm256); + } + } + + #[test] + fn two_pair_rotation_last_wins_on_writes() { + let raw = format!("aes-256-gcm:{K1_B64},aes-256-gcm:{K2_B64}"); + let pairs = parse_encryption_pair_list("t", &raw).unwrap(); + assert_eq!(pairs.len(), 2); + // Head pair (the write pair) is the LAST one — this test + // pins that invariant. When K2 wires the head-pair + // helpers, `pairs.last()` MUST resolve to K2's material. + let head = pairs.last().unwrap(); + assert_eq!(head.cipher, CipherKind::AesGcm256); + // Materials differ. + assert_ne!(pairs[0].key_material, pairs[1].key_material); + } + + #[test] + fn none_alone_is_legal_and_equivalent_to_unencrypted() { + let pairs = parse_encryption_pair_list("t", "none:").unwrap(); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].cipher, CipherKind::None); + assert!(pairs[0].key_material.is_none()); + } + + #[test] + fn none_first_then_aes_is_encrypt_migration_shape() { + // `none:,aes:K` — head is aes, writes now encrypt. Legacy + // plaintext blobs still read via the `none` pair while the + // rotation job walks them. + let raw = format!("none:,aes-256-gcm:{K1_B64}"); + let pairs = parse_encryption_pair_list("t", &raw).unwrap(); + assert_eq!(pairs.len(), 2); + assert_eq!(pairs[0].cipher, CipherKind::None); + assert_eq!(pairs[1].cipher, CipherKind::AesGcm256); + assert!(pairs[1].key_material.is_some()); + } + + #[test] + fn aes_first_then_none_is_decrypt_migration_shape() { + // `aes:K,none:` — head is none, writes now produce plaintext. + let raw = format!("aes-256-gcm:{K1_B64},none:"); + let pairs = parse_encryption_pair_list("t", &raw).unwrap(); + assert_eq!(pairs.len(), 2); + assert_eq!(pairs[0].cipher, CipherKind::AesGcm256); + assert_eq!(pairs[1].cipher, CipherKind::None); + } + + #[test] + fn whitespace_around_separators_is_tolerated() { + let raw = format!(" aes-256-gcm : {K1_B64} , none: "); + let pairs = parse_encryption_pair_list("t", &raw).unwrap(); + assert_eq!(pairs.len(), 2); + assert_eq!(pairs[0].cipher, CipherKind::AesGcm256); + assert_eq!(pairs[1].cipher, CipherKind::None); + } + + #[test] + fn empty_input_rejected() { + for raw in ["", " ", "\t\n "] { + let err = parse_encryption_pair_list("t", raw).unwrap_err(); + assert!(err.contains("empty"), "raw={raw:?} err={err}"); + } + } + + #[test] + fn leading_or_trailing_comma_rejected() { + for raw in [ + format!(",aes-256-gcm:{K1_B64}"), + format!("aes-256-gcm:{K1_B64},"), + format!("aes-256-gcm:{K1_B64},,aes-256-gcm:{K2_B64}"), + ] { + let err = parse_encryption_pair_list("t", &raw).unwrap_err(); + assert!(err.contains("empty pair"), "raw={raw:?} err={err}"); + } + } + + #[test] + fn unknown_cipher_rejected() { + let err = parse_encryption_pair_list("t", &format!("chacha20:{K1_B64}")).unwrap_err(); + assert!(err.contains("unknown cipher"), "err was: {err}"); + assert!(err.contains("chacha20"), "err was: {err}"); + } + + #[test] + fn none_with_material_rejected() { + // `none:` — nonsense. Must be `none:` (empty + // material after the colon). + let err = parse_encryption_pair_list("t", &format!("none:{K1_B64}")).unwrap_err(); + assert!( + err.contains("cipher `none` but has key material"), + "err was: {err}" + ); + } + + #[test] + fn multiple_none_pairs_rejected() { + let err = parse_encryption_pair_list("t", "none:,none:").unwrap_err(); + assert!(err.contains("more than one `none` pair"), "err was: {err}"); + } + + #[test] + fn empty_material_for_real_cipher_rejected() { + let err = parse_encryption_pair_list("t", "aes-256-gcm:").unwrap_err(); + assert!(err.contains("empty key material"), "err was: {err}"); + } + + #[test] + fn non_base64_key_rejected() { + let err = parse_encryption_pair_list("t", "aes-256-gcm:not_base64!!").unwrap_err(); + assert!(err.contains("not valid base64"), "err was: {err}"); + } + + #[test] + fn wrong_length_key_rejected() { + // "AAAA" decodes to 3 bytes — valid base64, wrong length. + let err = parse_encryption_pair_list("t", "aes-256-gcm:AAAA").unwrap_err(); + assert!(err.contains("32 bytes"), "err was: {err}"); + } + + #[test] + fn duplicate_key_material_rejected() { + let raw = format!("aes-256-gcm:{K1_B64},aes-256-gcm:{K1_B64}"); + let err = parse_encryption_pair_list("t", &raw).unwrap_err(); + assert!(err.contains("same key material twice"), "err was: {err}"); + } + + #[test] + fn entry_name_appears_in_error_message() { + let err = parse_encryption_pair_list("s3_prod", "").unwrap_err(); + assert!(err.contains("s3_prod"), "err was: {err}"); + } + + #[test] + fn fingerprint_is_12_hex_chars_for_real_cipher_and_none_for_none() { + let pairs = + parse_encryption_pair_list("t", &format!("aes-256-gcm:{K1_B64},none:")).unwrap(); + let fp0 = pairs[0].fingerprint_short().unwrap(); + assert_eq!(fp0.len(), 12); + assert!(fp0.chars().all(|c| c.is_ascii_hexdigit())); + assert!(pairs[1].fingerprint_short().is_none()); + } + + #[test] + fn fingerprint_stable_across_calls() { + let pairs_a = parse_encryption_pair_list("t", K1_B64).unwrap(); + let pairs_b = parse_encryption_pair_list("t", K1_B64).unwrap(); + assert_eq!( + pairs_a[0].fingerprint_short(), + pairs_b[0].fingerprint_short() + ); + } + + #[test] + fn fingerprint_differs_between_different_keys() { + let pairs = parse_encryption_pair_list( + "t", + &format!("aes-256-gcm:{K1_B64},aes-256-gcm:{K2_B64}"), + ) + .unwrap(); + assert_ne!(pairs[0].fingerprint_short(), pairs[1].fingerprint_short()); + } + } } From e16468977170c0168c79364663bf1d813e892a35 Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sat, 1 Aug 2026 21:59:48 +0200 Subject: [PATCH 03/29] feat(storage key rot): remove dead born OXICLOUD_STORAGE__ENCRYPTION_CIPHER + alway ovewrite on storage migration (got issue when migrating with blob already existing and a key change) --- docs/config/env.md | 8 +- docs/guide/backend-storage.md | 65 +++- example.env | 26 +- .../services/storage_settings_service.rs | 2 +- src/common/config.rs | 355 ++++++++++-------- src/common/di.rs | 2 +- src/infrastructure/services/entry_backend.rs | 34 +- .../services/storage_migration_service.rs | 61 +-- 8 files changed, 332 insertions(+), 221 deletions(-) diff --git a/docs/config/env.md b/docs/config/env.md index c39651e7..05febfdb 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -103,8 +103,7 @@ Each declared name `` then reads its own set of per-entry variables: | `OXICLOUD_STORAGE__AZURE_CONTAINER` | — | Azure-only: blob container name (required when backend=azure) | | `OXICLOUD_STORAGE__AZURE_SAS_TOKEN` | — | Azure-only: SAS token (alternative to account key) | | `OXICLOUD_STORAGE__AZURE_ENDPOINT_URL` | — | Azure-only: custom endpoint (Azurite, private deployments) | -| `OXICLOUD_STORAGE__ENCRYPTION_KEY` | — | Base64-encoded 32-byte AES-256 key. **Presence implies encryption is enabled** on this entry — no separate enable flag. Bad base64 / wrong length aborts boot. | -| `OXICLOUD_STORAGE__ENCRYPTION_CIPHER` | `aes-256-gcm` when `_ENCRYPTION_KEY` is set | Cipher choice for this entry. Only `aes-256-gcm` is accepted today (future-proofing knob — the enum is ready for a second cipher, the implementation still hardcodes AES-256-GCM). Setting the cipher without a key aborts boot. | +| `OXICLOUD_STORAGE__ENCRYPTION_KEY` | — | Comma-separated list of `:` pairs (or bare ``, which defaults to `aes-256-gcm`). **Presence implies encryption is enabled** on this entry — no separate enable flag. The LAST pair wins on writes; every pair is a candidate for reads. Supported ciphers: `aes-256-gcm` and `none` (empty-key sentinel used at pair-list head/tail for encrypt/decrypt-in-place rotations). Bad base64, wrong length, duplicate keys, or multiple `none` pairs abort boot. See [Storage key rotation](../plan/storage-key-rotation.md) for rotation recipes. | **Fail-fast rules** (boot aborts with actionable message): @@ -125,7 +124,10 @@ OXICLOUD_STORAGE_s3_prod_S3_BUCKET=my-oxicloud-bucket OXICLOUD_STORAGE_s3_prod_S3_REGION=us-east-1 OXICLOUD_STORAGE_s3_prod_S3_ACCESS_KEY=… OXICLOUD_STORAGE_s3_prod_S3_SECRET_KEY=… -OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=… # openssl rand -base64 32 +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:… # openssl rand -base64 32 + +# Rotation window (two pairs, last wins on writes): +# OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: ``` ## Storage Backend (DEPRECATED — legacy single-backend) diff --git a/docs/guide/backend-storage.md b/docs/guide/backend-storage.md index 6735711a..19d04bdd 100644 --- a/docs/guide/backend-storage.md +++ b/docs/guide/backend-storage.md @@ -40,15 +40,78 @@ A few things to know: Any backend can be encrypted at rest by adding an encryption key to it: ``` -OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=… # base64 of 32 random bytes +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:… # base64 of 32 random bytes ``` A key can be generated from **Settings → Storage → Generate key** in the admin panel. Set the same key on a new backend during migration and OxiCloud re-encrypts the data as it copies. +The bare shorthand `OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=` (no `aes-256-gcm:` prefix) also works and is treated as AES-256-GCM. Use the explicit form once you have more than one key in the list — see [Rotating an encryption key](#rotating-an-encryption-key) below. + ::: warning If you lose the encryption key, the data encrypted with it is unrecoverable. Store the key somewhere as safe as you'd store a database backup. ::: +### Rotating an encryption key + +When it's time to rotate a key — after a suspected leak, on a periodic policy, or after a staff turnover — you don't need to provision a second bucket. Add the new key **alongside** the old one; the server writes with the new key while reads keep working through the old one; then a background job re-encrypts existing data under the new key; then you drop the old key. + +Step by step: + +1. Generate a new key: + + ``` + openssl rand -base64 32 + ``` + +2. Append it to the entry's key list. **Order matters — the NEW key goes LAST.** + + ``` + OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: + ``` + +3. Restart the server. New uploads are now encrypted with the new key; existing files still open normally because the old key is still in the list. + +4. On the **Storage** tab, click **Rotate encryption key** on the entry. This dispatches a background job that re-encrypts every existing file under the new key. All operations keep working during rotation — uploads, browsing, downloads, sharing. + +5. Wait for the job to complete. Progress shows in a top banner and on the **Jobs** tab. + +6. Once the entry card says "*Rotation complete — safe to remove the old key*", remove the OLD key from the list: + + ``` + OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm: + ``` + +7. Restart the server. Rotation is done. + +::: warning +Do NOT remove the old key before the rotation job reports "safe to remove". Any file still encrypted under the old key would become unreadable. +::: + +### Encrypting a backend that started unencrypted + +Same shape as key rotation, using the `none:` sentinel to represent "the current writes are plaintext": + +1. Generate a new key. +2. Add it AFTER `none:`: + + ``` + OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY=none:,aes-256-gcm: + ``` + +3. Restart. New uploads are encrypted; existing plaintext files stay readable. +4. Trigger **Rotate encryption key** on the entry. +5. Once the entry card says the rotation is done, remove `none:` from the list; restart. + +### Decrypting an encrypted backend + +The symmetric flow — add `none:` AFTER the current key, restart, rotate, drop the old key: + +``` +OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY=aes-256-gcm:,none: +``` + +After the rotation job completes and the entry card says done, remove the AES pair (or the whole variable) and restart. All files on that entry are now plaintext. + ## Checking a backend Once a backend is declared, it appears on the admin **Storage** tab as a card: diff --git a/example.env b/example.env index f84b531a..45b9b06e 100644 --- a/example.env +++ b/example.env @@ -371,9 +371,16 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud # `OXICLOUD_STORAGE__BACKEND` (local | s3 | azure) plus the # backend-specific fields below. A missing required field aborts # boot with the exact var name in the error message. -# * Presence of `OXICLOUD_STORAGE__ENCRYPTION_KEY` implies AES-256 -# encryption is enabled on that entry (no separate enable flag). -# Bad base64 / wrong length aborts boot with the entry name. +# * `OXICLOUD_STORAGE__ENCRYPTION_KEY` is a comma-separated LIST +# of `:` pairs. Presence implies encryption is +# enabled on that entry (no separate enable flag). The LAST pair +# wins on writes; every pair is a candidate for reads. Supported +# ciphers: `aes-256-gcm` (default when no cipher prefix is given) +# and `none` (empty-key sentinel used at pair-list head for a +# decrypt-in-place rotation, or at tail for an encrypt-in-place +# rotation). Bad base64 / wrong length / duplicate keys / +# multiple `none` pairs abort boot with the entry name. See +# `docs/plan/storage-key-rotation.md` for rotation recipes. # * SETTING `_ENTRIES` alongside the legacy flat vars below (e.g. # `OXICLOUD_STORAGE_BACKEND` + `OXICLOUD_S3_BUCKET`) is a FAIL-FAST # boot error — pick one mode. Migrate any leftover flat vars into @@ -393,11 +400,14 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud #OXICLOUD_STORAGE_s3_prod_S3_ACCESS_KEY= #OXICLOUD_STORAGE_s3_prod_S3_SECRET_KEY= #OXICLOUD_STORAGE_s3_prod_S3_FORCE_PATH_STYLE=false -#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY= # generate: openssl rand -base64 32 -# Cipher declaration — future-proofing. Today only `aes-256-gcm` is -# accepted (and it's the default when `_ENCRYPTION_KEY` is set), so -# this line can be omitted. Explicit here as documentation. -#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_CIPHER=aes-256-gcm +#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm: # generate: openssl rand -base64 32 +# The bare shorthand `` (no `aes-256-gcm:` prefix) +# also works; the explicit form makes the cipher visible and is +# required once you have more than one pair in the list. +# +# Two-pair rotation example (paste both pairs in .env, restart, +# then trigger the format-upgrade job, then drop the OLD pair): +#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: # # Repair flag: if you rename an entry in .env while the DB still points # at the old name, boot aborts with an actionable error pointing at: diff --git a/src/application/services/storage_settings_service.rs b/src/application/services/storage_settings_service.rs index a96af68e..672df02b 100644 --- a/src/application/services/storage_settings_service.rs +++ b/src/application/services/storage_settings_service.rs @@ -273,7 +273,7 @@ impl StorageSettingsService { StorageBackendType::Azure => "azure".to_string(), }, is_active: e.name == active_entry_name, - encryption_enabled: e.encryption_key_base64.is_some(), + encryption_enabled: e.is_encrypted(), location_hint: entry_location_hint(e), }) .collect(); diff --git a/src/common/config.rs b/src/common/config.rs index ee53b08b..e68ae87f 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -383,80 +383,38 @@ impl Default for RetryConfig { } } -/// AES-GCM / AEAD cipher choice for a `NamedStorageEntry`. -/// -/// Today the only shipping variant is `Aes256Gcm` — the same cipher -/// `EncryptedBlobBackend` has always hardcoded. The enum is -/// future-proofing so `OXICLOUD_STORAGE__ENCRYPTION_CIPHER` is -/// already an accepted knob when a second cipher lands -/// (chacha20-poly1305, aes-256-siv, …). Absent + `_ENCRYPTION_KEY` -/// present → defaults to `Aes256Gcm` for back-compat with entries -/// declared before this field existed. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum EncryptionCipher { - /// AES-256 in Galois/Counter Mode. 96-bit nonce + 128-bit tag, - /// random nonce per blob. Layout on disk / S3: - /// `[12-byte nonce] [ciphertext] [16-byte GCM tag]`. - Aes256Gcm, -} - -impl EncryptionCipher { - /// Stable env-var value → variant. Case-insensitive. Extend with - /// new variants as new ciphers land; the surface is one match - /// arm here + one branch in `EncryptedBlobBackend::new_for_cipher` - /// (when that gets added). - pub fn parse(raw: &str) -> Option { - match raw.to_ascii_lowercase().as_str() { - "aes-256-gcm" | "aes256gcm" => Some(EncryptionCipher::Aes256Gcm), - _ => None, - } - } - - /// Stable env-var-friendly name — the exact string - /// `OXICLOUD_STORAGE__ENCRYPTION_CIPHER` accepts, and what - /// admin surfaces render back to operators. - pub fn as_str(self) -> &'static str { - match self { - EncryptionCipher::Aes256Gcm => "aes-256-gcm", - } - } -} - // ───────────────────────────────────────────────────────────────────── -// K1: pair-list encryption config (post-storage-multi-entry, pre-v1-header). -// See `docs/plan/storage-key-rotation.md`. +// K1: pair-list encryption config (post-storage-multi-entry, +// pre-v1-header). See `docs/plan/storage-key-rotation.md`. // -// The types and parser below REPLACE the singular `EncryptionCipher` + -// `encryption_key_base64` + `encryption_cipher` model with a -// list-of-pairs model where the last pair wins on writes and every -// pair is a candidate for reads. The `none` cipher is a first-class -// citizen so plaintext ↔ encrypted transitions can be expressed as a -// single pair-list evolution. +// [`CipherKind`], [`KeyPair`] and [`parse_encryption_pair_list`] +// replace the singular pre-K1 `EncryptionCipher` + `encryption_key_base64` +// + `encryption_cipher` model with a list-of-pairs model where the +// last pair wins on writes and every pair is a candidate for reads. +// The `none` cipher is a first-class citizen so plaintext ↔ encrypted +// transitions can be expressed as a single pair-list evolution. // -// K1 introduces these types but does NOT yet wire them into -// `NamedStorageEntry` — that flip lands in K2 alongside the v1 header -// read/write paths. The parser is exercised via unit tests only in -// this slice. +// K1.2 wires the pair-list into `NamedStorageEntry`; K2 replaces the +// on-disk format (`.blob` → v1 header at same suffix) and the read +// path (magic-byte dispatch + `` lookup). See the plan. // ───────────────────────────────────────────────────────────────────── /// The AEAD (or absence of one) used by a single [`KeyPair`]. /// -/// Distinct from the older [`EncryptionCipher`] enum in two ways: -/// -/// * Adds a `None` variant. A pair with `CipherKind::None` says +/// * `AesGcm256` — the only real AEAD OxiCloud ships today. On the +/// wire (v1 header): `[12-byte nonce] [ciphertext] [16-byte tag]`. +/// * `None` — no cipher. A pair with `CipherKind::None` says /// "writes routed to this pair produce raw plaintext". This is /// what makes the encrypt-a-plaintext-deployment and /// decrypt-an-encrypted-deployment recipes expressible without a /// second storage entry — see `docs/plan/storage-key-rotation.md` /// §"Encrypting a previously-plaintext deployment" and the /// symmetric decrypt recipe. -/// * Is the type carried by the v1 on-disk header's cipher field -/// (indirectly via `` in v1, but explicitly if a future -/// v2 adds a cipher byte to the header — the enum grows without -/// touching call sites). /// -/// Kept alongside [`EncryptionCipher`] during K1; K2 retires the -/// older enum in the same slice that flips `NamedStorageEntry`. +/// The type is carried by the v1 on-disk header's cipher field +/// (indirectly via `` in v1, but explicitly if a future +/// v2 adds a cipher byte to the header — the enum grows without +/// touching call sites). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CipherKind { /// AES-256 in Galois/Counter Mode. 96-bit nonce + 128-bit tag. @@ -473,9 +431,9 @@ pub enum CipherKind { } impl CipherKind { - /// Parse an env-var token: `"aes-256-gcm"` (with the - /// `aes256gcm` alias kept for continuity with - /// [`EncryptionCipher::parse`]) or `"none"`. Case-insensitive. + /// Parse an env-var token: `"aes-256-gcm"` (with the `aes256gcm` + /// alias kept for continuity with the pre-K1 spelling) or + /// `"none"`. Case-insensitive. pub fn parse(raw: &str) -> Option { match raw.to_ascii_lowercase().as_str() { "aes-256-gcm" | "aes256gcm" => Some(CipherKind::AesGcm256), @@ -695,6 +653,84 @@ pub fn parse_encryption_pair_list(entry_name: &str, raw: &str) -> Result { + tracing::info!( + target: "oxicloud::storage", + entry = %entry.name, + "storage entry `{}` — unencrypted", entry.name + ); + } + Some(pairs) => { + let head_idx = pairs.len() - 1; + let rendered: Vec = pairs + .iter() + .enumerate() + .map(|(i, kp)| { + let fp = kp.fingerprint_short().unwrap_or_else(|| "—".to_string()); + let head_mark = if i == head_idx { " ← head" } else { "" }; + format!("{}:{}{}", kp.cipher.as_str(), fp, head_mark) + }) + .collect(); + + tracing::info!( + target: "oxicloud::storage", + entry = %entry.name, + pairs = pairs.len(), + "storage entry `{}` — {} pair(s): {}", + entry.name, + pairs.len(), + rendered.join(", ") + ); + + // Warn on head-vs-other fingerprint divergence — the + // signal for "rotation in progress, don't drop the + // older key yet". Uses fingerprints (not raw + // material) so the comparison stays cheap and the + // log line reveals no key bytes. + let head_fp = pairs[head_idx].fingerprint_short(); + let non_head_fps: Vec> = pairs[..head_idx] + .iter() + .map(|kp| kp.fingerprint_short()) + .collect(); + if !non_head_fps.is_empty() && non_head_fps.iter().any(|fp| fp != &head_fp) { + tracing::warn!( + target: "oxicloud::storage", + entry = %entry.name, + head_fp = ?head_fp, + "storage entry `{}` has a rotation window open — head pair differs \ + from at least one older pair. Existing blobs written under an older \ + key stay readable, but keep the older pair in `.env` until the \ + format-upgrade job has reconciled them.", + entry.name + ); + } + } + } + } +} + /// One named storage entry declared in `.env`. /// /// See `docs/plan/storage-multi-entry.md`. Each entry is a fully-realised @@ -724,22 +760,67 @@ pub struct NamedStorageEntry { pub s3: Option, /// Azure configuration for `Azure`. `None` for other backends. pub azure: Option, - /// Per-entry AES-256-GCM key (base64, exactly 32 bytes decoded). - /// Presence implies encryption is enabled on this entry — no - /// separate `_ENCRYPTION_ENABLED` toggle. Absence = raw backend. - /// Validated at parse time; boot aborts on invalid key. - pub encryption_key_base64: Option, - /// Cipher choice for this entry. `Some(Aes256Gcm)` today when - /// `_ENCRYPTION_KEY` is set (whether or not the operator - /// declared `_ENCRYPTION_CIPHER=aes-256-gcm` explicitly, since - /// that's currently the only supported variant). - /// `None` when no encryption key is set. When new ciphers land, - /// the parser reads `_ENCRYPTION_CIPHER` to distinguish; today - /// the field is effectively a boolean because there's exactly - /// one variant. Kept as an enum on the type so downstream code - /// pattern-matches the concept explicitly and we don't lose the - /// intent when a second cipher is added. - pub encryption_cipher: Option, + /// Ordered list of `:` pairs from + /// `OXICLOUD_STORAGE__ENCRYPTION_KEY`. `None` means the + /// operator did not set the variable — the entry is + /// unencrypted, writes and reads pass through the raw backend. + /// + /// `Some(pairs)` is guaranteed non-empty (parser rejects the + /// empty-list case). The LAST pair is the write pair; every + /// pair is a candidate for reads. See + /// `docs/plan/storage-key-rotation.md` §"The pair-list config". + /// + /// Access this field through the [`Self::head_key_material`], + /// [`Self::head_cipher`], [`Self::is_encrypted`], and + /// [`Self::encryption_pairs`] helpers rather than pattern-matching + /// directly — they encapsulate the "unencrypted vs `none:`-headed + /// pair-list" distinction and keep call sites stable across + /// future K2 changes. + pub encryption: Option>, +} + +impl NamedStorageEntry { + /// Head pair — the one used for writes (K1) and, once K2 wires + /// the header, for read-dispatch of blobs whose header advertises + /// the head pair's `key_fp`. + /// + /// `None` when the entry has no `_ENCRYPTION_KEY` at all OR when + /// the head pair is `none:` (writes produce plaintext, so no key + /// material to hand to the AEAD). + pub fn head_key_material(&self) -> Option<&[u8; 32]> { + self.encryption + .as_ref() + .and_then(|pairs| pairs.last()) + .and_then(|kp| kp.key_material.as_ref()) + } + + /// Head pair's cipher choice. `None` when the entry has no + /// `_ENCRYPTION_KEY` at all. `Some(CipherKind::None)` when the + /// head pair is explicitly `none:` — semantically distinct from + /// "unconfigured", useful during decrypt-in-place migrations. + pub fn head_cipher(&self) -> Option { + self.encryption + .as_ref() + .and_then(|pairs| pairs.last()) + .map(|kp| kp.cipher) + } + + /// `true` iff writes to this entry produce ciphertext right now. + /// Distinct from "the operator has ever configured encryption" — + /// during a decrypt-in-place migration this returns `false` (head + /// is `none:`) even though older pairs in the list are real + /// ciphers used to READ existing encrypted blobs. + pub fn is_encrypted(&self) -> bool { + self.head_cipher().is_some_and(|c| c != CipherKind::None) + } + + /// The whole pair list, or an empty slice when the entry is + /// unencrypted. Used by K2's read path to walk pairs and by + /// `storage_rotate` to enumerate legacy pairs. Callers that only + /// need the write pair should prefer [`Self::head_key_material`]. + pub fn encryption_pairs(&self) -> &[KeyPair] { + self.encryption.as_deref().unwrap_or(&[]) + } } /// Validation for a `NamedStorageEntry.name`. Restricts to a safe subset @@ -989,56 +1070,22 @@ fn parse_named_entry(name: &str) -> Result { } } - // Encryption key — presence implies enabled. Validate now (base64 + - // decoded length) so we fail at boot, not at first blob write. - let encryption_key_base64 = match env::var(format!("OXICLOUD_STORAGE_{name}_ENCRYPTION_KEY")) { - Ok(k) if !k.is_empty() => { - validate_encryption_key(name, &k)?; - Some(k) - } + // Encryption — pair-list format. Parser accepts both the K1 + // shape (`aes-256-gcm:` or bare ``) and future shapes + // (2-pair rotation, `none:` head, …). See + // `docs/plan/storage-key-rotation.md`. + let encryption = match env::var(format!("OXICLOUD_STORAGE_{name}_ENCRYPTION_KEY")) { + Ok(raw) if !raw.trim().is_empty() => Some(parse_encryption_pair_list(name, &raw)?), _ => None, }; - // Cipher — future-proofing knob. Only `aes-256-gcm` today. - // Explicit value → parse (unknown = fail-fast so a typo isn't - // silently defaulted). Absent + key set → default to - // `Aes256Gcm`. Absent + no key → `None` (no encryption at all). - let encryption_cipher = match env::var(format!("OXICLOUD_STORAGE_{name}_ENCRYPTION_CIPHER")) { - Ok(raw) if !raw.is_empty() => match EncryptionCipher::parse(&raw) { - Some(c) => Some(c), - None => { - return Err(format!( - "OXICLOUD_STORAGE_{name}_ENCRYPTION_CIPHER=`{raw}` is not a known cipher — \ - supported: `aes-256-gcm`." - )); - } - }, - _ => { - if encryption_key_base64.is_some() { - Some(EncryptionCipher::Aes256Gcm) - } else { - None - } - } - }; - - // Nonsense combo — cipher declared without a key. Refuse rather - // than silently ignore the operator's declared intent. - if encryption_cipher.is_some() && encryption_key_base64.is_none() { - return Err(format!( - "OXICLOUD_STORAGE_{name}_ENCRYPTION_CIPHER is set but \ - OXICLOUD_STORAGE_{name}_ENCRYPTION_KEY is not — set the key too, or remove the cipher." - )); - } - Ok(NamedStorageEntry { name: name.to_string(), backend, root_dir, s3, azure, - encryption_key_base64, - encryption_cipher, + encryption, }) } @@ -1104,20 +1151,18 @@ fn synthesize_default_from_legacy_vars() -> Result { } } - let encryption_key_base64 = match env::var("OXICLOUD_STORAGE_ENCRYPTION_KEY") { - Ok(k) if !k.is_empty() => { - validate_encryption_key("default", &k)?; - Some(k) - } + // Legacy synthesis path: the pre-multi-entry world had a single + // `OXICLOUD_STORAGE_ENCRYPTION_KEY` (raw base64, no cipher/none + // syntax). The pair-list parser accepts that shape as a + // 1-pair `aes-256-gcm:` after defaulting the cipher, so we + // route through it and get the same validation for free. + // + // The legacy flat surface has no cipher variable, so no + // agreement check is needed here. + let encryption = match env::var("OXICLOUD_STORAGE_ENCRYPTION_KEY") { + Ok(k) if !k.trim().is_empty() => Some(parse_encryption_pair_list("default", &k)?), _ => None, }; - // Legacy synthesis: no `OXICLOUD_STORAGE_ENCRYPTION_CIPHER` env - // var exists in the legacy flat surface — always default to - // AES-256-GCM when a legacy key is set (matches the hardcoded - // pre-multi-entry behaviour). - let encryption_cipher = encryption_key_base64 - .as_ref() - .map(|_| EncryptionCipher::Aes256Gcm); Ok(NamedStorageEntry { name: "default".to_string(), @@ -1125,33 +1170,10 @@ fn synthesize_default_from_legacy_vars() -> Result { root_dir, s3, azure, - encryption_key_base64, - encryption_cipher, + encryption, }) } -/// Validate a base64-encoded 32-byte encryption key. Called at parse -/// time (not first-use) so a bad key aborts boot with a clear -/// per-entry message instead of failing much later inside -/// `EncryptedBlobBackend::new`. -fn validate_encryption_key(entry_name: &str, key_b64: &str) -> Result<(), String> { - use base64::Engine; - let decoded = base64::engine::general_purpose::STANDARD - .decode(key_b64) - .map_err(|e| { - format!("OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY is not valid base64: {e}") - })?; - if decoded.len() != 32 { - return Err(format!( - "OXICLOUD_STORAGE_{entry_name}_ENCRYPTION_KEY decodes to {} bytes; must be exactly \ - 32 bytes (AES-256). Generate a fresh key via \ - `POST /api/admin/settings/storage/generate-key`.", - decoded.len() - )); - } - Ok(()) -} - impl Default for StorageConfig { fn default() -> Self { // Architecture-appropriate max upload size to avoid overflow on 32-bit systems @@ -2907,6 +2929,7 @@ impl AppConfig { config.storage_entries = parse_storage_entries().unwrap_or_else(|e| { panic!("Invalid storage configuration in environment: {e}"); }); + log_storage_encryption_summary(&config.storage_entries); // Storage backend selection if let Ok(backend) = env::var("OXICLOUD_STORAGE_BACKEND") { @@ -3405,7 +3428,7 @@ mod tests { assert_eq!(entries[0].backend, StorageBackendType::Local); assert_eq!(entries[0].root_dir.as_deref(), Some("/data")); assert!(entries[0].s3.is_none()); - assert!(entries[0].encryption_key_base64.is_none()); + assert!(entries[0].encryption.is_none()); } #[test] @@ -3433,10 +3456,12 @@ mod tests { set("OXICLOUD_STORAGE_ENCRYPTION_KEY", VALID_KEY_B64); let entries = parse_storage_entries().unwrap(); assert_eq!(entries.len(), 1); - assert_eq!( - entries[0].encryption_key_base64.as_deref(), - Some(VALID_KEY_B64) - ); + // Legacy flat-var path routes through `parse_encryption_pair_list` + // and produces a 1-pair `aes-256-gcm:` list. + let pairs = entries[0].encryption.as_ref().expect("expected pair list"); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].cipher, CipherKind::AesGcm256); + assert_eq!(pairs[0].key_material, Some([0u8; 32])); } #[test] @@ -3492,10 +3517,10 @@ mod tests { set("OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY", VALID_KEY_B64); let entries = parse_storage_entries().unwrap(); assert_eq!(entries.len(), 1); - assert_eq!( - entries[0].encryption_key_base64.as_deref(), - Some(VALID_KEY_B64) - ); + let pairs = entries[0].encryption.as_ref().expect("expected pair list"); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].cipher, CipherKind::AesGcm256); + assert_eq!(pairs[0].key_material, Some([0u8; 32])); } // ── Cell (Yes, Yes) — fail fast on conflict diff --git a/src/common/di.rs b/src/common/di.rs index ca555373..5565483c 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -366,7 +366,7 @@ impl AppServiceFactory { // // When `storage_entries` is non-empty, encryption is already // applied inside `build_entry_backend` from the entry's own - // `encryption_key_base64` (per-entry key). This block is the + // pair-list (head-pair key). This block is the // pre-multi-entry fallback that reads the flat // `OXICLOUD_STORAGE_ENCRYPTION_*` vars — reachable only for // fresh installs with no explicit storage config at all diff --git a/src/infrastructure/services/entry_backend.rs b/src/infrastructure/services/entry_backend.rs index fa427f21..0c915256 100644 --- a/src/infrastructure/services/entry_backend.rs +++ b/src/infrastructure/services/entry_backend.rs @@ -229,30 +229,24 @@ pub fn build_entry_backend( } }; - // Encryption decorator — presence-implies-enabled, per plan §Encryption. - let Some(key_b64) = entry.encryption_key_base64.as_ref() else { + // Encryption decorator — presence-implies-enabled, per plan + // §Encryption. K1 preserves single-head-pair behaviour: writes and + // reads use the head pair's material. K2 replaces the decorator + // with a header-aware read/write path that consults the full pair + // list; this call site becomes a wrapper construction rather than + // a single-key handoff at that point. + let Some(key) = entry.head_key_material() else { + // No pair list at all, OR head pair is `CipherKind::None` + // (mid-decrypt-migration state). Both cases mean "writes go + // straight to the raw backend today"; older encrypted pairs + // in the list stay unreachable until K2 wires the read + // fallback. return base; }; use crate::infrastructure::services::encrypted_blob_backend::EncryptedBlobBackend; - let key_bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, key_b64) - .unwrap_or_else(|e| { - panic!( - "entry `{}` encryption key is not valid base64: {e} — parser was supposed to \ - catch this at boot", - entry.name - ) - }); - let key: [u8; 32] = key_bytes.try_into().unwrap_or_else(|v: Vec| { - panic!( - "entry `{}` encryption key decoded to {} bytes; must be 32 — parser was supposed to \ - catch this at boot", - entry.name, - v.len() - ) - }); tracing::info!( - "Storage entry `{}` encrypted with AES-256-GCM (key from env)", + "Storage entry `{}` encrypted with AES-256-GCM (head pair from env)", entry.name ); - Arc::new(EncryptedBlobBackend::new(base, &key)) + Arc::new(EncryptedBlobBackend::new(base, key)) } diff --git a/src/infrastructure/services/storage_migration_service.rs b/src/infrastructure/services/storage_migration_service.rs index 6436ed47..49390df8 100644 --- a/src/infrastructure/services/storage_migration_service.rs +++ b/src/infrastructure/services/storage_migration_service.rs @@ -328,7 +328,10 @@ impl RecoverableJobHandler for StorageMigrationService { if let Some(source) = source_entry && entry_identity(source) == entry_identity(target_entry) { - let key_differs = source.encryption_key_base64 != target_entry.encryption_key_base64; + // Compare head-pair materials — the "write key" for each + // entry. A non-head pair difference (mid-rotation) doesn't + // count as a key change for the purposes of this refusal. + let key_differs = source.head_key_material() != target_entry.head_key_material(); let hint = if key_differs { " (encryption key differs → this looks like an in-place key rotation; \ create a new entry pointing at a DIFFERENT bucket / dir, migrate to it, \ @@ -446,7 +449,14 @@ impl RecoverableJobHandler for StorageMigrationService { }; let mut copied_count = 0u64; - let mut skipped_count = 0u64; + // K1.2: with the target-skip short-circuit gone (see the + // detailed comment further down), no blob is ever "skipped" + // during a migration walk today. The counter stays wired + // through the log lines + `finish_completed` so K3's + // format-aware smart-skip can re-populate it without + // touching the observability surface. Not mutated in this + // slice — hence no `mut`. + let skipped_count: u64 = 0; let mut failed_count = 0u64; let mut source_missing_count = 0u64; @@ -574,26 +584,33 @@ impl RecoverableJobHandler for StorageMigrationService { } } - // Skip when the target already has it — supports - // idempotent resume and cheap re-runs against a - // partially-migrated target. - match target.blob_exists(hash).await { - Ok(true) => { - skipped_count += 1; - continue; - } - Ok(false) => {} - Err(e) => { - tracing::warn!( - target: "oxicloud::migration", - event = "storage_migration.blob_exists_error", - run_id = %store.run_id(), - hash = %hash, - error = %e, - "blob_exists probe on target failed; attempting copy anyway" - ); - } - } + // Always copy — do NOT short-circuit on + // `target.blob_exists(hash)`. Ed hit this on 2026-08-01 + // during S3 → local migration testing with encryption + // enabled on the target: the target had pre-existing + // plaintext blobs from an earlier local-active session, + // so `blob_exists` returned true and the migration + // silently skipped them. Result: the "encrypted" + // target ended up with mixed plaintext + ciphertext + // blobs — undetectable until a subsequent read failed. + // + // The old skip was justified by two use cases: + // (a) resume idempotency — the last cursor-checkpoint + // window (~100 blobs) gets re-processed on resume; + // (b) target-side dedup — same content already present. + // + // Both are now handled by unconditional overwrite: the + // re-copy is bounded by the checkpoint window (small), + // and dedup-hit content is rare in practice + // (content-addressability means duplicate blobs ARE + // the same blob unless two backends were seeded + // separately from the same source). + // + // K3's `storage_rotate` job will restore a smart skip + // via the v1 header's `` field — "already at + // head format+key" then becomes cheaply detectable + // without reading target bytes. Until then, correct > + // fast. match copy_blob(self.source.as_ref(), target.as_ref(), hash).await { Ok(()) => { From 9485ee554081b7dae09348ad5e0a1d9adfe8db2d Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sat, 1 Aug 2026 23:06:31 +0200 Subject: [PATCH 04/29] feat(storage key rot): add blob header engine --- examples/bench_s3_put.rs | 2 +- src/common/config.rs | 109 +++ src/common/di.rs | 2 +- src/infrastructure/services/dedup_service.rs | 2 +- .../services/encrypted_blob_backend.rs | 668 +++++++++++++++--- src/infrastructure/services/entry_backend.rs | 35 +- 6 files changed, 714 insertions(+), 104 deletions(-) diff --git a/examples/bench_s3_put.rs b/examples/bench_s3_put.rs index 19902152..3e32f416 100644 --- a/examples/bench_s3_put.rs +++ b/examples/bench_s3_put.rs @@ -353,7 +353,7 @@ async fn main() { // Full production composition: Cache(Encrypted(Retry(S3))). let cache_dir_b = tempfile::tempdir().expect("tempdir"); let full_stack: Arc = Arc::new(CachedBlobBackend::new( - Arc::new(EncryptedBlobBackend::new( + Arc::new(EncryptedBlobBackend::new_single_aes( Arc::new(RetryBlobBackend::new( backend.clone() as Arc, RetryPolicy::default(), diff --git a/src/common/config.rs b/src/common/config.rs index e68ae87f..ea1eff85 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -483,6 +483,30 @@ pub struct KeyPair { } impl KeyPair { + /// Construct a real-cipher pair with pre-decoded key material. + /// Convenience for tests + the pre-multi-entry legacy synthesis + /// path where the base64-decoded key is already in hand. New + /// production code paths get their pairs from + /// [`parse_encryption_pair_list`] which produces the same + /// shape. + pub fn new_aes_gcm(key: [u8; 32]) -> Self { + Self { + cipher: CipherKind::AesGcm256, + key_material: Some(key), + } + } + + /// Construct a `none:` sentinel pair. Used by `entry_backend.rs` + /// under the always-wrap rule to synthesise a 1-pair list for + /// entries with no `_ENCRYPTION_KEY` declared, so those writes + /// still get v1 headers (plaintext-v1 flavor). + pub fn new_none() -> Self { + Self { + cipher: CipherKind::None, + key_material: None, + } + } + /// Truncated SHA-256 fingerprint of the key material — 12 hex /// chars (6 bytes of SHA output). Used at boot for the audit- /// line dump so operators can eyeball which key is at each @@ -501,6 +525,41 @@ impl KeyPair { let full = Sha256::digest(mat); Some(hex::encode(&full[..6])) } + + /// 8-byte SHA-256 truncation used as the v1 header's `` + /// field on every encrypted blob. Read dispatch looks up the + /// matching [`KeyPair`] in the pair list by this value in O(1), + /// so a blob written under any pair in the list can be decrypted + /// without falling through candidate keys. + /// + /// Semantic contract with the on-disk format: + /// + /// * `CipherKind::AesGcm256` pair → 8 bytes of `sha256(key)[..8]`. + /// Effectively unique per configured pair (2⁻⁶⁴ collision on + /// random keys — never in practice on the ~1-3 pairs a real + /// deployment has). + /// * `CipherKind::None` pair → all-zero. This is what marks a + /// plaintext-v1 blob so the read path can dispatch "return + /// post-header raw bytes" without consulting a key. Real + /// ciphers cannot collide with all-zero: an + /// `sha256(key)[..8] == 0` real key is 2⁻⁶⁴ improbable AND + /// the parser wouldn't accept two pairs with the same fp + /// (uniqueness check on raw key material — same key = same + /// fp). + /// + /// Distinct truncation from [`Self::fingerprint_short`] on + /// purpose: this is a raw byte string embedded in every + /// encrypted blob (compactness matters), while the boot log + /// wants a legible short-hex string. + pub fn key_fp(&self) -> [u8; 8] { + use sha2::{Digest, Sha256}; + let mut fp = [0u8; 8]; + if let Some(mat) = self.key_material.as_ref() { + let full = Sha256::digest(mat); + fp.copy_from_slice(&full[..8]); + } + fp + } } /// Parse the `OXICLOUD_STORAGE__ENCRYPTION_KEY` env var value @@ -3860,5 +3919,55 @@ mod tests { .unwrap(); assert_ne!(pairs[0].fingerprint_short(), pairs[1].fingerprint_short()); } + + // ── key_fp (8-byte header field) tests ──────────────────── + + #[test] + fn key_fp_is_eight_bytes() { + let pairs = parse_encryption_pair_list("t", K1_B64).unwrap(); + let fp = pairs[0].key_fp(); + assert_eq!(fp.len(), 8); + // At least one byte non-zero (K1 = 0x00..0x1F sha256 has + // ample entropy; if this ever asserts we've got a truly + // improbable collision). + assert!(fp.iter().any(|b| *b != 0), "fp = {fp:?}"); + } + + #[test] + fn key_fp_zero_for_none_cipher() { + let pairs = parse_encryption_pair_list("t", "none:").unwrap(); + assert_eq!(pairs[0].key_fp(), [0u8; 8]); + } + + #[test] + fn key_fp_stable_across_calls() { + let pairs_a = parse_encryption_pair_list("t", K1_B64).unwrap(); + let pairs_b = parse_encryption_pair_list("t", K1_B64).unwrap(); + assert_eq!(pairs_a[0].key_fp(), pairs_b[0].key_fp()); + } + + #[test] + fn key_fp_differs_between_different_keys() { + let pairs = parse_encryption_pair_list( + "t", + &format!("aes-256-gcm:{K1_B64},aes-256-gcm:{K2_B64}"), + ) + .unwrap(); + assert_ne!(pairs[0].key_fp(), pairs[1].key_fp()); + } + + #[test] + fn key_fp_and_fingerprint_short_share_prefix() { + // Both derive from the same sha256(key); the on-blob fp is + // 8 raw bytes, the log fp is hex of the first 6. Pinning + // this alignment protects against a future refactor that + // accidentally switches one to a different hash / offset. + let pairs = parse_encryption_pair_list("t", K1_B64).unwrap(); + let hex_fp = pairs[0].fingerprint_short().unwrap(); + let raw_fp = pairs[0].key_fp(); + // First 12 hex chars of the log fp = hex of the first 6 + // bytes of the raw fp. + assert_eq!(&hex_fp[..12], &hex::encode(&raw_fp[..6])); + } } } diff --git a/src/common/di.rs b/src/common/di.rs index 5565483c..8a3e253e 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -387,7 +387,7 @@ impl AppServiceFactory { let key: [u8; 32] = key_bytes.try_into().expect( "OXICLOUD_STORAGE_ENCRYPTION_KEY must be exactly 32 bytes (base64 of 32 bytes)", ); - blob_backend = Arc::new(EncryptedBlobBackend::new(blob_backend, &key)); + blob_backend = Arc::new(EncryptedBlobBackend::new_single_aes(blob_backend, &key)); tracing::info!("Blob storage encryption decorator enabled (AES-256-GCM) — legacy path"); } diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 6339bbbb..7c4e10a6 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -3672,7 +3672,7 @@ mod rechunk_integration_tests { let inner = Arc::new(LocalBlobBackend::new(&dir.path().join("blobs"))); inner.initialize().await.expect("init backend"); let key = EncryptedBlobBackend::generate_key(); - let backend = Arc::new(EncryptedBlobBackend::new(inner, &key)); + let backend = Arc::new(EncryptedBlobBackend::new_single_aes(inner, &key)); DedupService::new(backend, pool.clone(), pool.clone()) } diff --git a/src/infrastructure/services/encrypted_blob_backend.rs b/src/infrastructure/services/encrypted_blob_backend.rs index d4d3de05..e38a0497 100644 --- a/src/infrastructure/services/encrypted_blob_backend.rs +++ b/src/infrastructure/services/encrypted_blob_backend.rs @@ -1,16 +1,62 @@ -//! `EncryptedBlobBackend` — AES-256-GCM encryption decorator for blob storage. +//! `EncryptedBlobBackend` — v1 blob-format decorator for blob storage. //! -//! Transparently encrypts blobs before they reach the inner backend and -//! decrypts them on read. Each blob gets a random 96-bit nonce which is -//! prepended to the ciphertext. The GCM authentication tag (16 bytes) is -//! appended by the cipher. +//! Wraps an inner `BlobStorageBackend` and adds two orthogonal +//! behaviours: +//! +//! 1. **v1 header framing on every new write** — every blob written +//! by this wrapper starts with a 15-byte header +//! `OXCPT | | ` so future reads are +//! self-describing regardless of the entry's current config. +//! 2. **Pair-list encryption** — the wrapper owns an ordered +//! [`KeyPair`] list. Writes use the LAST pair (the "head"); reads +//! dispatch on the header's `` field into an O(1) +//! fp → cipher lookup, so a blob written under any pair still in +//! the list decrypts without fallback attempts. +//! +//! See `docs/plan/storage-key-rotation.md` for the full design. +//! +//! ## v1 on-disk layout +//! +//! Encrypted-v1 (`head_cipher` is a real AEAD): +//! +//! ```text +//! "OXCPT" 5 bytes — magic marker +//! 2 bytes — big-endian u16; v1 = 0x0001 +//! 8 bytes — sha256(key material)[..8], routes reads +//! 12 bytes — random per blob (AES-GCM 96-bit) +//! N bytes — same length as plaintext +//! 16 bytes — AEAD authentication tag +//! ``` +//! +//! Plaintext-v1 (`head_cipher` is `None`, i.e. entry uses a `none:` +//! head pair or has no encryption declared at all): +//! +//! ```text +//! "OXCPT" 5 bytes — magic marker +//! 2 bytes — big-endian u16; v1 = 0x0001 +//! 8 bytes — all zero +//! N bytes — raw plaintext +//! ``` +//! +//! ## Legacy fallback on reads +//! +//! Blobs written before this wrapper existed have no OXCPT magic. +//! Reads check the first 5 bytes: +//! +//! * `"OXCPT"` → v1 path (version + key_fp lookup + AEAD or raw). +//! * anything else → **legacy path** — try `head_cipher` (if any) +//! as an AES-GCM decode over the pre-v1 shape +//! `[nonce][ciphertext][tag]`; otherwise return raw bytes. +//! +//! Collision probability: 2⁻⁴⁰ per blob for random legacy bytes to +//! start with `"OXCPT"`. If it happens, subsequent version / key_fp +//! checks fail with a hard error (`UnsupportedBlobVersion` or +//! `NoKeyForBlob`) — never silent misread. //! //! **IMPORTANT**: BLAKE3 hashing is performed on the *plaintext* by //! `DedupService` before this layer sees the blob, so content-addressable //! dedup still works correctly. //! -//! Layout on disk/S3: `[12-byte nonce][ciphertext + 16-byte GCM tag]` -//! //! ## Runtime & memory characteristics //! //! GCM is all-or-nothing per blob: a blob can only be decrypted whole, so @@ -27,6 +73,7 @@ //! the ciphertext buffer is reused for the plaintext instead of allocating //! a second copy. +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::pin::Pin; @@ -39,14 +86,42 @@ use tokio::fs; use crate::application::ports::blob_storage_ports::{ BlobStorageBackend, BlobStream, StorageHealthStatus, }; +use crate::common::config::KeyPair; use crate::domain::errors::DomainError; +/// v1 magic marker — every v1 blob starts with these 5 ASCII bytes. +/// Chosen for its 2⁻⁴⁰ collision odds against random legacy data +/// and its clean legibility in a `hexdump` (mnemonic: +/// "OXiCloud CiPherText"). +const OXCPT_MAGIC: &[u8; 5] = b"OXCPT"; + +/// v1 header version bytes (big-endian u16 = 0x0001). Future formats +/// bump this in-place — the [`OXCPT_MAGIC`] stays the discriminator +/// against legacy / non-OxiCloud files. +const V1_VERSION_BYTES: [u8; 2] = [0x00, 0x01]; + +/// On-blob `` field size — 8 bytes = 64-bit truncation of +/// `sha256(key)`. Wide enough that random collisions are 2⁻⁶⁴; the +/// parser also uniqueness-checks pairs on raw key material so +/// duplicates can't sneak in. +const KEY_FP_SIZE: usize = 8; + +/// Total v1 header size = magic + version + key_fp. +const HEADER_SIZE: usize = 5 + 2 + KEY_FP_SIZE; + /// Nonce size for AES-256-GCM (96 bits = 12 bytes). const NONCE_SIZE: usize = 12; /// AES-256-GCM authentication tag length appended after the ciphertext. const TAG_SIZE: usize = 16; +/// AEAD overhead per blob (nonce + tag) — 28 bytes, same regardless +/// of header framing. +const AEAD_OVERHEAD: usize = NONCE_SIZE + TAG_SIZE; + +/// Per-blob overhead for an encrypted-v1 blob: header + AEAD = 43 bytes. +const ENCRYPTED_V1_OVERHEAD: usize = HEADER_SIZE + AEAD_OVERHEAD; + /// Payloads at or above this size run crypto on the blocking pool; below /// it the `spawn_blocking` round-trip costs more than the AES work itself. const CRYPTO_OFFLOAD_THRESHOLD: usize = 64 * 1024; @@ -56,23 +131,82 @@ const CRYPTO_OFFLOAD_THRESHOLD: usize = 64 * 1024; /// hashers) see the same backpressure shape either way. const PLAINTEXT_EMIT_SIZE: usize = 64 * 1024; -/// `BlobStorageBackend` decorator that encrypts blobs at rest. +/// `BlobStorageBackend` decorator that applies v1 header framing and +/// pair-list-driven encryption. See the module-level docs for the +/// on-disk layout and read-fallback semantics. pub struct EncryptedBlobBackend { inner: Arc, - /// `Arc` so the per-op `clone()` handed to `offload_crypto` closures is - /// an atomic bump instead of copying the ~240-byte expanded AES-256 - /// round-key schedule on every chunk read/write. - cipher: Arc, + /// The pair list as declared by the operator. Guaranteed + /// non-empty by ctor (an empty input auto-synthesises a single + /// `none:` pair, so the invariant holds). Preserved verbatim for + /// observability and for downstream K3 `storage_rotate` which + /// needs to walk pair indices. + #[allow(dead_code)] + pairs: Vec, + /// `` → per-pair cipher, for O(1) read dispatch on v1 + /// blobs. Excludes any `none:` pair (nothing to build). Cloned + /// per read via `Arc::clone` — the ~240-byte expanded AES-256 + /// round-key schedule is amortised across every request. + fp_ciphers: HashMap<[u8; KEY_FP_SIZE], Arc>, + /// Cipher used by writes (last pair in the list). `None` when + /// the head is a `CipherKind::None` pair — in that case writes + /// emit plaintext-v1 (magic + version + zero fp + raw payload). + head_cipher: Option>, + /// Head pair's `key_fp` — embedded in every write's v1 header. + /// `[0u8; 8]` when head is `CipherKind::None`, matching the + /// plaintext-v1 shape. + head_key_fp: [u8; KEY_FP_SIZE], } impl EncryptedBlobBackend { - /// Create a new encryption layer wrapping `inner`. + /// Primary constructor. Takes an ordered pair list — the LAST + /// pair is the write pair (head), every pair is a candidate for + /// reads via `` dispatch. /// - /// `key` must be exactly 32 bytes (AES-256). - pub fn new(inner: Arc, key: &[u8; 32]) -> Self { - let cipher = - Arc::new(Aes256Gcm::new_from_slice(key).expect("AES-256 key must be 32 bytes")); - Self { inner, cipher } + /// Empty `pairs` is legal and treated as a single implicit + /// `none:` pair — the wrapper still emits v1 headers on writes + /// (plaintext-v1 flavor) and still magic-byte-dispatches on + /// reads (with legacy fallback for header-less blobs). This is + /// the always-wrap contract used by `entry_backend.rs` under the + /// K2 "normalize data" rule. + /// + /// Panics if any real-cipher pair's key material isn't 32 bytes, + /// which the parser guarantees — a panic here signals a + /// programmer bug, not operator error. + pub fn new(inner: Arc, pairs: Vec) -> Self { + let pairs = if pairs.is_empty() { + vec![KeyPair::new_none()] + } else { + pairs + }; + let mut fp_ciphers = HashMap::with_capacity(pairs.len()); + for kp in &pairs { + if let Some(mat) = kp.key_material.as_ref() { + let cipher = Aes256Gcm::new_from_slice(mat) + .expect("KeyPair invariant: real-cipher pair has 32-byte key"); + fp_ciphers.insert(kp.key_fp(), Arc::new(cipher)); + } + } + let head = pairs + .last() + .expect("post-normalisation pair list is non-empty"); + let head_key_fp = head.key_fp(); + let head_cipher = fp_ciphers.get(&head_key_fp).cloned(); + Self { + inner, + pairs, + fp_ciphers, + head_cipher, + head_key_fp, + } + } + + /// Convenience: wrap with a single AES-256-GCM pair. Same effect + /// as `new(inner, vec![KeyPair::new_aes_gcm(*key)])`. Used by + /// tests + the pre-multi-entry legacy synthesis fallback in + /// `di.rs`. + pub fn new_single_aes(inner: Arc, key: &[u8; 32]) -> Self { + Self::new(inner, vec![KeyPair::new_aes_gcm(*key)]) } /// Generate a random 32-byte key suitable for AES-256. @@ -84,27 +218,132 @@ impl EncryptedBlobBackend { } } -/// Encrypt `data` into the on-disk layout: `[12-byte nonce][ciphertext + tag]`. +/// Assemble an encrypted-v1 blob: +/// `OXCPT | v1 | key_fp | nonce | ciphertext | tag`. /// -/// Single output buffer, mirroring the read side's in-place detached decrypt: -/// the payload is copied exactly once and encrypted in place with the tag -/// appended. The old shape let `cipher.encrypt` allocate a full ciphertext -/// `Vec` and then copied it a second time behind the nonce — one extra -/// allocation + a full-size memcpy on every encrypted chunk write -/// (benches/ROUND11.md §15; output bytes identical for a given nonce). -fn encrypt_bytes(cipher: &Aes256Gcm, data: &[u8]) -> Result { +/// Single output buffer, mirroring the read side's in-place detached +/// decrypt: the payload is copied exactly once and encrypted in +/// place with the tag appended. The old shape (pre-K2, no header) +/// let `cipher.encrypt` allocate a full ciphertext `Vec` and then +/// copied it a second time behind the nonce — one extra allocation + +/// a full-size memcpy on every encrypted chunk write +/// (benches/ROUND11.md §15). K2 preserves the single-buffer +/// discipline: we `extend_from_slice` header + nonce + payload, then +/// encrypt in place from `HEADER_SIZE + NONCE_SIZE`. +fn encrypt_v1( + cipher: &Aes256Gcm, + head_key_fp: [u8; KEY_FP_SIZE], + data: &[u8], +) -> Result { let nonce = Aes256Gcm::generate_nonce(&mut OsRng); - let mut out = Vec::with_capacity(NONCE_SIZE + data.len() + TAG_SIZE); + let mut out = Vec::with_capacity(ENCRYPTED_V1_OVERHEAD + data.len()); + out.extend_from_slice(OXCPT_MAGIC); + out.extend_from_slice(&V1_VERSION_BYTES); + out.extend_from_slice(&head_key_fp); out.extend_from_slice(nonce.as_slice()); out.extend_from_slice(data); let tag = cipher - .encrypt_in_place_detached(&nonce, b"", &mut out[NONCE_SIZE..]) + .encrypt_in_place_detached(&nonce, b"", &mut out[HEADER_SIZE + NONCE_SIZE..]) .map_err(|e| DomainError::internal_error("Encryption", format!("encrypt failed: {e}")))?; out.extend_from_slice(&tag); Ok(Bytes::from(out)) } -/// Decrypt the on-disk layout `[nonce][ciphertext + tag]` **in place**. +/// Assemble a plaintext-v1 blob: +/// `OXCPT | v1 | <8 zero bytes> | payload`. +/// +/// No crypto, no allocation beyond the header prefix. Produced when +/// the wrapper's head pair is `CipherKind::None`. +fn write_plaintext_v1(data: &[u8]) -> Bytes { + let mut out = Vec::with_capacity(HEADER_SIZE + data.len()); + out.extend_from_slice(OXCPT_MAGIC); + out.extend_from_slice(&V1_VERSION_BYTES); + out.extend_from_slice(&[0u8; KEY_FP_SIZE]); + out.extend_from_slice(data); + Bytes::from(out) +} + +/// Read dispatch — the K2 core. Given a fetched blob and the +/// wrapper's pair table, returns plaintext. +/// +/// * `OXCPT` at position 0 → **v1 path**: +/// * version check (only `0x0001` accepted today); +/// * `key_fp == [0u8; 8]` → plaintext-v1 → return post-header +/// bytes as-is; +/// * else `key_fp` lookup in `fp_ciphers` → AEAD decrypt over +/// the post-header body. +/// * anything else → **legacy path**: +/// * `head_cipher = Some` → AES-GCM decrypt over pre-K2 shape +/// (`nonce | ct | tag`) — the pre-v1 world had exactly one key +/// per entry, and that key is the head pair by construction of +/// any pair-list upgraded from a pre-K1 config; +/// * `head_cipher = None` → return raw bytes (pre-K2 plaintext +/// deployment). +/// +/// Never falls through silently: every failure returns a distinct +/// typed error (`UnsupportedBlobVersion`, `NoKeyForBlob`, AEAD tag +/// failure). Random legacy bytes matching `OXCPT` (2⁻⁴⁰) fail the +/// subsequent version/key_fp check with a hard error, not silent +/// garbage. +fn read_dispatch( + fp_ciphers: &HashMap<[u8; KEY_FP_SIZE], Arc>, + head_cipher: Option<&Aes256Gcm>, + encrypted: Vec, +) -> Result { + if encrypted.len() >= 5 && &encrypted[..5] == OXCPT_MAGIC { + return read_v1(fp_ciphers, encrypted); + } + match head_cipher { + Some(cipher) => decrypt_aead_in_place(cipher, encrypted), + None => Ok(Bytes::from(encrypted)), + } +} + +/// The v1 branch of `read_dispatch`, factored out for clarity. +fn read_v1( + fp_ciphers: &HashMap<[u8; KEY_FP_SIZE], Arc>, + encrypted: Vec, +) -> Result { + if encrypted.len() < HEADER_SIZE { + return Err(DomainError::internal_error( + "Encryption", + format!( + "v1 blob too short (need at least {HEADER_SIZE} bytes for the header, got {})", + encrypted.len() + ), + )); + } + let version = &encrypted[5..7]; + if version != V1_VERSION_BYTES { + return Err(DomainError::internal_error( + "Encryption", + format!( + "unsupported v1 blob version 0x{:02x}{:02x} — this build only reads 0x0001", + version[0], version[1] + ), + )); + } + let mut key_fp = [0u8; KEY_FP_SIZE]; + key_fp.copy_from_slice(&encrypted[7..HEADER_SIZE]); + let mut body = encrypted; + body.drain(..HEADER_SIZE); + if key_fp == [0u8; KEY_FP_SIZE] { + // Plaintext-v1 — post-header bytes ARE the plaintext. + return Ok(Bytes::from(body)); + } + let cipher = fp_ciphers.get(&key_fp).ok_or_else(|| { + DomainError::internal_error( + "Encryption", + format!( + "v1 blob key_fp {} does not match any configured pair — cannot decrypt", + hex::encode(key_fp) + ), + ) + })?; + decrypt_aead_in_place(cipher, body) +} + +/// Decrypt the AEAD body `[nonce][ciphertext][tag]` **in place**. /// /// Reuses the encrypted buffer for the plaintext, so peak RAM is one buffer — /// not ciphertext + plaintext side by side (which for legacy whole-file blobs @@ -112,18 +351,12 @@ fn encrypt_bytes(cipher: &Aes256Gcm, data: &[u8]) -> Result /// are lifted to the stack, the ciphertext body is decrypted in place via the /// detached API (mirroring the encrypt side's `encrypt_in_place_detached`), and /// the plaintext is returned as a zero-copy `Bytes::slice` past the nonce. -/// -/// The prior shape did `encrypted.split_off(NONCE_SIZE)`, which allocated a -/// fresh `Vec` and memcpy'd the entire ciphertext (up to a whole legacy blob) -/// on every decrypted read — one full-payload allocation + copy the doc comment -/// above claimed did not happen (benches/ROUND25.md §M1; ROUND11 §15 fixed only -/// the encrypt side). Output plaintext is byte-identical. -fn decrypt_bytes(cipher: &Aes256Gcm, mut encrypted: Vec) -> Result { +fn decrypt_aead_in_place(cipher: &Aes256Gcm, mut encrypted: Vec) -> Result { let len = encrypted.len(); - if len < NONCE_SIZE + TAG_SIZE { + if len < AEAD_OVERHEAD { return Err(DomainError::internal_error( "Encryption", - "encrypted blob too short (missing nonce/tag)", + "AEAD body too short (missing nonce/tag)", )); } // Nonce (first 12 bytes) and GCM tag (last 16 bytes) copied to the stack so @@ -186,21 +419,15 @@ impl BlobStorageBackend for EncryptedBlobBackend { let inner = self.inner.clone(); let hash = hash.to_string(); let source = source_path.to_path_buf(); - let cipher = self.cipher.clone(); + let head_cipher = self.head_cipher.clone(); + let head_key_fp = self.head_key_fp; Box::pin(async move { // Read plaintext from source let plaintext = fs::read(&source).await.map_err(|e| { DomainError::internal_error("Encryption", format!("read source: {e}")) })?; - - let len = plaintext.len(); - let encrypted = offload_crypto(len, move || encrypt_bytes(&cipher, &plaintext)).await?; - - // Hand the ciphertext straight to the inner backend. The previous - // implementation spooled it to a `.enc.tmp` file only for the - // inner backend to read it back — a full extra write + read of - // every blob that came through this path. - inner.put_blob_from_bytes(&hash, encrypted).await + let out = frame_write(head_cipher, head_key_fp, plaintext).await?; + inner.put_blob_from_bytes(&hash, out).await }) } @@ -211,11 +438,11 @@ impl BlobStorageBackend for EncryptedBlobBackend { ) -> Pin> + Send + '_>> { let inner = self.inner.clone(); let hash = hash.to_string(); - let cipher = self.cipher.clone(); + let head_cipher = self.head_cipher.clone(); + let head_key_fp = self.head_key_fp; Box::pin(async move { - let encrypted = - offload_crypto(data.len(), move || encrypt_bytes(&cipher, data.as_ref())).await?; - inner.put_blob_from_bytes(&hash, encrypted).await + let out = frame_write(head_cipher, head_key_fp, data.to_vec()).await?; + inner.put_blob_from_bytes(&hash, out).await }) } @@ -226,11 +453,11 @@ impl BlobStorageBackend for EncryptedBlobBackend { ) -> Pin> + Send + '_>> { let inner = self.inner.clone(); let hash = hash.to_string(); - let cipher = self.cipher.clone(); + let head_cipher = self.head_cipher.clone(); + let head_key_fp = self.head_key_fp; Box::pin(async move { - let encrypted = - offload_crypto(data.len(), move || encrypt_bytes(&cipher, data.as_ref())).await?; - inner.put_blob_from_bytes_unsynced(&hash, encrypted).await + let out = frame_write(head_cipher, head_key_fp, data.to_vec()).await?; + inner.put_blob_from_bytes_unsynced(&hash, out).await }) } @@ -250,14 +477,18 @@ impl BlobStorageBackend for EncryptedBlobBackend { { let inner = self.inner.clone(); let hash = hash.to_string(); - let cipher = self.cipher.clone(); + let fp_ciphers = self.fp_ciphers.clone(); + let head_cipher = self.head_cipher.clone(); Box::pin(async move { - // GCM must see the whole message: collect ciphertext, decrypt in - // place off the runtime, then stream zero-copy plaintext slices. + // Collect the full blob, dispatch on magic bytes off the + // runtime, then stream zero-copy plaintext slices. let enc_stream = inner.get_blob_stream(&hash).await?; let encrypted = collect_stream(enc_stream).await?; let len = encrypted.len(); - let plaintext = offload_crypto(len, move || decrypt_bytes(&cipher, encrypted)).await?; + let plaintext = offload_crypto(len, move || { + read_dispatch(&fp_ciphers, head_cipher.as_deref(), encrypted) + }) + .await?; Ok(plaintext_stream(plaintext)) }) } @@ -271,16 +502,20 @@ impl BlobStorageBackend for EncryptedBlobBackend { { let inner = self.inner.clone(); let hash = hash.to_string(); - let cipher = self.cipher.clone(); + let fp_ciphers = self.fp_ciphers.clone(); + let head_cipher = self.head_cipher.clone(); Box::pin(async move { - // Decrypt the full blob, then slice the plaintext range without - // copying. For CDC chunks (every blob written since chunking - // landed) this is ≤ 1 MiB; only legacy whole-file blobs pay a - // full-blob decrypt here — see the module docs. + // Decrypt (or unwrap) the full blob, then slice the plaintext + // range without copying. For CDC chunks (every blob written + // since chunking landed) this is ≤ 1 MiB; only legacy whole-file + // blobs pay a full-blob decrypt here — see the module docs. let enc_stream = inner.get_blob_stream(&hash).await?; let encrypted = collect_stream(enc_stream).await?; let len = encrypted.len(); - let plaintext = offload_crypto(len, move || decrypt_bytes(&cipher, encrypted)).await?; + let plaintext = offload_crypto(len, move || { + read_dispatch(&fp_ciphers, head_cipher.as_deref(), encrypted) + }) + .await?; // `end` is exclusive — same contract as `LocalBlobBackend`, whose // implementation reads `end - start` bytes. The previous version @@ -313,14 +548,29 @@ impl BlobStorageBackend for EncryptedBlobBackend { &self, hash: &str, ) -> Pin> + Send + '_>> { - // The stored size includes nonce + GCM tag overhead. - // Return the *plaintext* size by subtracting overhead. + // Plaintext size = stored size - per-format overhead. The exact + // overhead depends on which format the blob is in (encrypted-v1 + // = 43, plaintext-v1 = 15, legacy-encrypted = 28, legacy-plain = + // 0), which we can't know without inspecting bytes. We assume + // the blob was written under the wrapper's current head — that's + // true for every new write from K2 onward. + // + // For legacy blobs still on disk the estimate is off by + // ±(HEADER_SIZE) or so. Since `blob_size` is used for capacity + // metrics and admin dashboards (not byte-exact accounting — + // content-hash is the source of truth for that), a small drift + // during the legacy-blob window is acceptable. If a hot path + // starts depending on byte-exact `blob_size`, revisit. let inner = self.inner.clone(); let hash = hash.to_string(); + let overhead = if self.head_cipher.is_some() { + ENCRYPTED_V1_OVERHEAD as u64 + } else { + HEADER_SIZE as u64 + }; Box::pin(async move { - let encrypted_size = inner.blob_size(&hash).await?; - // overhead = 12 (nonce) + 16 (GCM tag) = 28 bytes - Ok(encrypted_size.saturating_sub(28)) + let stored = inner.blob_size(&hash).await?; + Ok(stored.saturating_sub(overhead)) }) } @@ -330,16 +580,28 @@ impl BlobStorageBackend for EncryptedBlobBackend { Box> + Send + '_>, > { let inner = self.inner.clone(); + let (outer_name, cipher_desc) = if self.head_cipher.is_some() { + ("encrypted", "AES-256-GCM") + } else { + ("v1-plaintext", "none") + }; Box::pin(async move { let mut status = inner.health_check().await?; - status.backend_type = format!("encrypted({})", status.backend_type); - status.message = format!("{} | Encryption: AES-256-GCM", status.message); + status.backend_type = format!("{outer_name}({})", status.backend_type); + status.message = format!("{} | Encryption: {cipher_desc}", status.message); Ok(status) }) } fn backend_type(&self) -> &'static str { - "encrypted" + // Choice 2/B: dynamic — reflects head-pair semantics so admin + // surfaces show "v1-plaintext(local)" for a `none:`-headed + // entry instead of misleadingly saying "encrypted(local)". + if self.head_cipher.is_some() { + "encrypted" + } else { + "v1-plaintext" + } } /// Transparent wrapper: the inner backend serves the bytes. @@ -376,6 +638,24 @@ impl BlobStorageBackend for EncryptedBlobBackend { } } +/// Frame a plaintext payload into a v1 blob per the head-pair +/// configuration. Encrypted case runs the AEAD on the blocking pool +/// for large payloads; plaintext case is a small header-prepend and +/// stays inline (no crypto = no offload). +async fn frame_write( + head_cipher: Option>, + head_key_fp: [u8; KEY_FP_SIZE], + plaintext: Vec, +) -> Result { + match head_cipher { + Some(cipher) => { + let len = plaintext.len(); + offload_crypto(len, move || encrypt_v1(&cipher, head_key_fp, &plaintext)).await + } + None => Ok(write_plaintext_v1(&plaintext)), + } +} + /// Collect a byte stream into a single `Vec`. /// /// Modern blobs are CDC chunks (≤ `CDC_MAX_CHUNK` + nonce/tag overhead), @@ -419,7 +699,7 @@ mod tests { local.initialize().await.unwrap(); let key = EncryptedBlobBackend::generate_key(); - let encrypted = EncryptedBlobBackend::new(local, &key); + let encrypted = EncryptedBlobBackend::new_single_aes(local, &key); // Write a test blob let data = b"Hello, encrypted world!"; @@ -467,7 +747,7 @@ mod tests { local.initialize().await.unwrap(); let key = EncryptedBlobBackend::generate_key(); - let encrypted = EncryptedBlobBackend::new(local, &key); + let encrypted = EncryptedBlobBackend::new_single_aes(local, &key); // 300 KiB of a repeating pattern — crosses the offload threshold and // spans several PLAINTEXT_EMIT_SIZE slices. @@ -520,7 +800,7 @@ mod tests { local.initialize().await.unwrap(); let key = EncryptedBlobBackend::generate_key(); - let encrypted = EncryptedBlobBackend::new(local.clone(), &key); + let encrypted = EncryptedBlobBackend::new_single_aes(local.clone(), &key); let hash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; encrypted @@ -528,16 +808,20 @@ mod tests { .await .unwrap(); - // Corrupt one ciphertext byte on disk (past the 12-byte nonce). + // Corrupt one ciphertext byte on disk. v1 layout: 15-byte + // header + 12-byte nonce + ciphertext, so the first + // ciphertext byte is at `HEADER_SIZE + NONCE_SIZE`. Flipping + // it must fail AEAD tag verification. let path = local.local_blob_path(hash).expect("local path"); let mut raw = std::fs::read(&path).unwrap(); - raw[NONCE_SIZE] ^= 0xFF; + raw[HEADER_SIZE + NONCE_SIZE] ^= 0xFF; std::fs::write(&path, raw).unwrap(); assert!(encrypted.get_blob_stream(hash).await.is_err()); } - /// Decrypting with a different key must fail authentication. + /// Decrypting with a different key must fail — post-K2 via the + /// key_fp lookup (writer's fp isn't in the reader's map). #[tokio::test] async fn test_wrong_key_fails_decrypt() { let tmp = TempDir::new().unwrap(); @@ -545,14 +829,230 @@ mod tests { local.initialize().await.unwrap(); let hash = "aaaabbbbccccddddaaaabbbbccccddddaaaabbbbccccddddaaaabbbbccccdddd"; - let writer = - EncryptedBlobBackend::new(local.clone(), &EncryptedBlobBackend::generate_key()); + let writer = EncryptedBlobBackend::new_single_aes( + local.clone(), + &EncryptedBlobBackend::generate_key(), + ); writer .put_blob_from_bytes(hash, Bytes::from_static(b"locked")) .await .unwrap(); - let reader = EncryptedBlobBackend::new(local, &EncryptedBlobBackend::generate_key()); - assert!(reader.get_blob_stream(hash).await.is_err()); + let reader = + EncryptedBlobBackend::new_single_aes(local, &EncryptedBlobBackend::generate_key()); + // Result::unwrap_err needs Ok: Debug; BlobStream isn't Debug. + // Match directly, and extract the message off the DomainError. + let err = match reader.get_blob_stream(hash).await { + Ok(_) => panic!("expected a decrypt error, got Ok"), + Err(e) => e, + }; + let msg = err.to_string(); + assert!( + msg.contains("key_fp") || msg.contains("does not match"), + "expected NoKeyForBlob-shape error, got: {msg}" + ); + } + + // ───────────────────────────────────────────────────────────── + // K2 tests — v1 header format + magic-byte dispatch + legacy + // fallback + pair-list read routing. + // ───────────────────────────────────────────────────────────── + + /// Every encrypted-v1 blob starts with the magic + version + + /// head-pair fingerprint. Pins the on-disk byte layout so a + /// future refactor can't silently break the format. + #[tokio::test] + async fn v1_encrypted_blob_has_expected_header() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + let key = [42u8; 32]; + let backend = EncryptedBlobBackend::new_single_aes(local.clone(), &key); + + let hash = "1111111111111111111111111111111111111111111111111111111111111111"; + backend + .put_blob_from_bytes(hash, Bytes::from_static(b"hello world")) + .await + .unwrap(); + + let path = local.local_blob_path(hash).expect("local path"); + let raw = std::fs::read(&path).unwrap(); + assert!( + raw.len() >= ENCRYPTED_V1_OVERHEAD, + "blob too short: {}", + raw.len() + ); + assert_eq!(&raw[..5], OXCPT_MAGIC, "missing OXCPT magic"); + assert_eq!(&raw[5..7], &V1_VERSION_BYTES, "wrong version bytes"); + let expected_fp = KeyPair::new_aes_gcm(key).key_fp(); + assert_eq!(&raw[7..HEADER_SIZE], &expected_fp, "wrong key_fp in header"); + } + + /// `none:`-headed entry emits plaintext-v1 (header + raw + /// payload, no crypto). Round-trip must yield identity bytes. + #[tokio::test] + async fn v1_plaintext_blob_round_trips() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + let backend = EncryptedBlobBackend::new(local.clone(), vec![KeyPair::new_none()]); + + let hash = "2222222222222222222222222222222222222222222222222222222222222222"; + let payload = Bytes::from_static(b"cleartext bytes"); + backend + .put_blob_from_bytes(hash, payload.clone()) + .await + .unwrap(); + + // On-disk shape: magic + version + zero fp + raw payload. + let path = local.local_blob_path(hash).expect("local path"); + let raw = std::fs::read(&path).unwrap(); + assert_eq!(&raw[..5], OXCPT_MAGIC); + assert_eq!(&raw[5..7], &V1_VERSION_BYTES); + assert_eq!(&raw[7..HEADER_SIZE], &[0u8; KEY_FP_SIZE]); + assert_eq!(&raw[HEADER_SIZE..], payload.as_ref()); + + // Read must strip the header and return payload identity. + let stream = backend.get_blob_stream(hash).await.unwrap(); + let round_tripped = collect_stream(stream).await.unwrap(); + assert_eq!(round_tripped, payload.as_ref()); + } + + /// Legacy fallback: a blob written with the pre-K2 AEAD shape + /// (nonce | ct | tag, no OXCPT header) still decrypts via the + /// head-pair AES key. This is the guarantee that upgrading to + /// K2 doesn't break existing encrypted deployments. + #[tokio::test] + async fn legacy_encrypted_blob_still_readable() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + let key = [0x33u8; 32]; + let backend = EncryptedBlobBackend::new_single_aes(local.clone(), &key); + + // Craft a legacy blob by hand: AES-GCM with random nonce, + // no OXCPT header. Matches exactly what pre-K2 code wrote. + let plaintext = b"legacy secret"; + let cipher = Aes256Gcm::new_from_slice(&key).unwrap(); + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let mut legacy = Vec::new(); + legacy.extend_from_slice(nonce.as_slice()); + legacy.extend_from_slice(plaintext); + let tag = cipher + .encrypt_in_place_detached(&nonce, b"", &mut legacy[NONCE_SIZE..]) + .unwrap(); + legacy.extend_from_slice(&tag); + + // Write the raw bytes directly onto the local backend, bypassing + // the wrapper (else it'd add a v1 header). + local + .put_blob_from_bytes( + "3333333333333333333333333333333333333333333333333333333333333333", + Bytes::from(legacy), + ) + .await + .unwrap(); + + // The wrapper's read path must dispatch on absent magic → + // legacy branch → head-pair AES decode. + let stream = backend + .get_blob_stream("3333333333333333333333333333333333333333333333333333333333333333") + .await + .unwrap(); + let got = collect_stream(stream).await.unwrap(); + assert_eq!(got, plaintext); + } + + /// Legacy fallback for pure plaintext: entry has no encryption + /// (empty pair list → `none:` synthesised), reads of a raw-byte + /// blob written before the wrapper existed still return raw. + #[tokio::test] + async fn legacy_plaintext_blob_still_readable() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + let backend = EncryptedBlobBackend::new(local.clone(), vec![]); + + let hash = "4444444444444444444444444444444444444444444444444444444444444444"; + let raw = b"just some bytes, no header"; + // Bypass the wrapper — write raw plaintext directly. + local + .put_blob_from_bytes(hash, Bytes::from_static(raw)) + .await + .unwrap(); + + // No magic → legacy path → head is None → return bytes as-is. + let stream = backend.get_blob_stream(hash).await.unwrap(); + let got = collect_stream(stream).await.unwrap(); + assert_eq!(got, raw); + } + + /// Pair-list key rotation: write under pair[0]'s key, add + /// pair[1] as head, read must still succeed via pair[0]'s + /// key_fp entry in the lookup table. + #[tokio::test] + async fn read_dispatches_by_key_fp_in_pair_list() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + let k_old = [0x11u8; 32]; + let k_new = [0x22u8; 32]; + + // Write with a single-pair wrapper using k_old. + let writer = EncryptedBlobBackend::new_single_aes(local.clone(), &k_old); + let hash = "5555555555555555555555555555555555555555555555555555555555555555"; + writer + .put_blob_from_bytes(hash, Bytes::from_static(b"payload")) + .await + .unwrap(); + + // Reader has BOTH keys: k_old at position 0, k_new at head. + // The blob's key_fp field points at k_old → lookup succeeds + // even though writes now go under k_new. + let reader = EncryptedBlobBackend::new( + local, + vec![KeyPair::new_aes_gcm(k_old), KeyPair::new_aes_gcm(k_new)], + ); + let stream = reader.get_blob_stream(hash).await.unwrap(); + let got = collect_stream(stream).await.unwrap(); + assert_eq!(got, b"payload"); + } + + /// Malformed v1 blob (correct magic, unknown version bytes) → + /// hard error, never silent misread. Guards against the + /// theoretical 2⁻⁴⁰ magic-collision case on random legacy data. + #[tokio::test] + async fn unknown_v1_version_returns_hard_error() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + let backend = EncryptedBlobBackend::new(local.clone(), vec![]); + + let hash = "6666666666666666666666666666666666666666666666666666666666666666"; + // Magic OK, version = 0xFFFF (future format we don't know). + let mut bogus = Vec::from(*OXCPT_MAGIC); + bogus.extend_from_slice(&[0xFF, 0xFF]); + bogus.extend_from_slice(&[0u8; KEY_FP_SIZE]); + bogus.extend_from_slice(b"payload"); + local + .put_blob_from_bytes(hash, Bytes::from(bogus)) + .await + .unwrap(); + + let err = match backend.get_blob_stream(hash).await { + Ok(_) => panic!("expected an UnsupportedBlobVersion error, got Ok"), + Err(e) => e, + }; + let msg = err.to_string(); + assert!( + msg.contains("unsupported v1 blob version"), + "expected UnsupportedBlobVersion error, got: {msg}" + ); } } diff --git a/src/infrastructure/services/entry_backend.rs b/src/infrastructure/services/entry_backend.rs index 0c915256..932947d6 100644 --- a/src/infrastructure/services/entry_backend.rs +++ b/src/infrastructure/services/entry_backend.rs @@ -229,24 +229,25 @@ pub fn build_entry_backend( } }; - // Encryption decorator — presence-implies-enabled, per plan - // §Encryption. K1 preserves single-head-pair behaviour: writes and - // reads use the head pair's material. K2 replaces the decorator - // with a header-aware read/write path that consults the full pair - // list; this call site becomes a wrapper construction rather than - // a single-key handoff at that point. - let Some(key) = entry.head_key_material() else { - // No pair list at all, OR head pair is `CipherKind::None` - // (mid-decrypt-migration state). Both cases mean "writes go - // straight to the raw backend today"; older encrypted pairs - // in the list stay unreachable until K2 wires the read - // fallback. - return base; - }; + // v1 wrapper (Choice 1/B: always wrap). Every entry gets the + // header-aware read/write path — even entries with no + // `_ENCRYPTION_KEY` at all. This normalises the on-disk format + // going forward: all new writes carry the OXCPT v1 header, all + // reads magic-byte-dispatch (with legacy fallback for + // header-less pre-K2 blobs). Not backwards-compatible with + // pre-K2 code trying to read new writes — but Ed's called it: + // uniform format is worth the one-way door. use crate::infrastructure::services::encrypted_blob_backend::EncryptedBlobBackend; + let pairs = entry.encryption.clone().unwrap_or_default(); + let mode = match entry.head_cipher() { + Some(crate::common::config::CipherKind::AesGcm256) => "encrypted-v1", + _ => "plaintext-v1", + }; tracing::info!( - "Storage entry `{}` encrypted with AES-256-GCM (head pair from env)", - entry.name + "Storage entry `{}` — {} wrapper (pairs: {})", + entry.name, + mode, + pairs.len() ); - Arc::new(EncryptedBlobBackend::new(base, key)) + Arc::new(EncryptedBlobBackend::new(base, pairs)) } From 30b0000c261a94b7e9e47d9a1c5d6c77d02a76c9 Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sat, 1 Aug 2026 23:25:27 +0200 Subject: [PATCH 05/29] feat(storage key rot): add blob/chunk creation with the header --- .../services/encrypted_blob_backend.rs | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) diff --git a/src/infrastructure/services/encrypted_blob_backend.rs b/src/infrastructure/services/encrypted_blob_backend.rs index e38a0497..6f7ae38c 100644 --- a/src/infrastructure/services/encrypted_blob_backend.rs +++ b/src/infrastructure/services/encrypted_blob_backend.rs @@ -216,6 +216,101 @@ impl EncryptedBlobBackend { OsRng.fill_bytes(&mut key); key } + + /// The format `storage_rotate` should normalise every blob TO — + /// derived from the wrapper's head pair. When + /// `head_cipher.is_some()` we're writing encrypted-v1 with the + /// head pair's `key_fp`; when it's `None` we're writing + /// plaintext-v1 (all-zero `key_fp`). + /// + /// K3 uses this as the "target format" that a per-blob decision + /// tree compares against `BlobFormat::classify(bytes)` — any + /// mismatch means the blob needs rewriting. + pub fn head_format(&self) -> BlobFormat { + if self.head_cipher.is_some() { + BlobFormat::EncryptedV1 { + key_fp: self.head_key_fp, + } + } else { + BlobFormat::PlaintextV1 + } + } + + /// Fetch, classify, and decrypt a blob in one round-trip. Used by + /// K3's `storage_rotate` per-blob step: it needs both the + /// plaintext (to re-encrypt under the head pair) AND the current + /// on-disk format (to decide whether a rewrite is needed at all). + /// + /// The inner backend is read once; the raw bytes are inspected + /// for their format before being consumed by `read_dispatch`. No + /// duplicated I/O. + /// + /// Returned tuple: `(plaintext, current_format)`. Rotate compares + /// `current_format` against [`Self::head_format`]; equal → skip, + /// different → rewrite via the standard write path. + pub async fn read_and_classify(&self, hash: &str) -> Result<(Bytes, BlobFormat), DomainError> { + let enc_stream = self.inner.get_blob_stream(hash).await?; + let raw = collect_stream(enc_stream).await?; + let format = BlobFormat::classify(&raw); + let fp_ciphers = self.fp_ciphers.clone(); + let head_cipher = self.head_cipher.clone(); + let len = raw.len(); + let plaintext = offload_crypto(len, move || { + read_dispatch(&fp_ciphers, head_cipher.as_deref(), raw) + }) + .await?; + Ok((plaintext, format)) + } +} + +/// Classification of a raw blob's on-disk format. Exposed for K3's +/// `storage_rotate` decision tree; not used on the hot request path. +/// +/// PartialEq is derived so `current == head_format` collapses the +/// plan's six-case decision tree into a single equality check: +/// +/// * `Legacy != anything v1` → always rewrite. +/// * `EncryptedV1{fp_a} != EncryptedV1{fp_b}` when fps differ → rewrite (key rotation). +/// * `PlaintextV1 != EncryptedV1` and vice-versa → rewrite (encrypt / decrypt in place). +/// * Match cases → skip (already normalised). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlobFormat { + /// No `OXCPT` magic. Pre-K2 shape — either raw plaintext or raw + /// `nonce | ct | tag` AES-GCM output; the wrapper's legacy read + /// path handles both. + Legacy, + /// v1 with a `none:`-style all-zero `key_fp`. Post-header bytes + /// are raw plaintext. + PlaintextV1, + /// v1 with a real cipher pair. `key_fp` identifies which pair + /// (matches [`KeyPair::key_fp`]). + EncryptedV1 { key_fp: [u8; KEY_FP_SIZE] }, +} + +impl BlobFormat { + /// Inspect the first `HEADER_SIZE` bytes and classify the blob. + /// O(1), no allocation. Used by [`EncryptedBlobBackend::read_and_classify`] + /// but also useful in isolation for offline tools. + pub fn classify(bytes: &[u8]) -> Self { + if bytes.len() < 5 || &bytes[..5] != OXCPT_MAGIC { + return BlobFormat::Legacy; + } + // Magic OK. If the rest of the header isn't present the blob + // is malformed — treat as Legacy so the decision tree marks + // it for rewrite (and the actual read will surface the error + // to the finding stream). + if bytes.len() < HEADER_SIZE { + return BlobFormat::Legacy; + } + // v1 magic + at least a full header. `key_fp` == 0 → plaintext. + let mut key_fp = [0u8; KEY_FP_SIZE]; + key_fp.copy_from_slice(&bytes[7..HEADER_SIZE]); + if key_fp == [0u8; KEY_FP_SIZE] { + BlobFormat::PlaintextV1 + } else { + BlobFormat::EncryptedV1 { key_fp } + } + } } /// Assemble an encrypted-v1 blob: @@ -1055,4 +1150,120 @@ mod tests { "expected UnsupportedBlobVersion error, got: {msg}" ); } + + // ───────────────────────────────────────────────────────────── + // K3 tests — BlobFormat classifier + head_format + read_and_classify. + // + // These pin the format-inspection contract that `storage_rotate` + // depends on. The rotate job's per-blob decision tree collapses + // to `current != head_format ? rewrite : skip`, so any drift in + // either helper would silently change rotation semantics. + // ───────────────────────────────────────────────────────────── + + #[test] + fn classify_recognises_encrypted_v1() { + let mut blob = Vec::from(*OXCPT_MAGIC); + blob.extend_from_slice(&V1_VERSION_BYTES); + let key_fp = [0x11u8; KEY_FP_SIZE]; + blob.extend_from_slice(&key_fp); + blob.extend_from_slice(b"nonce_ct_tag_bytes_would_go_here"); + assert_eq!( + BlobFormat::classify(&blob), + BlobFormat::EncryptedV1 { key_fp } + ); + } + + #[test] + fn classify_recognises_plaintext_v1() { + let mut blob = Vec::from(*OXCPT_MAGIC); + blob.extend_from_slice(&V1_VERSION_BYTES); + blob.extend_from_slice(&[0u8; KEY_FP_SIZE]); + blob.extend_from_slice(b"raw payload after header"); + assert_eq!(BlobFormat::classify(&blob), BlobFormat::PlaintextV1); + } + + #[test] + fn classify_recognises_legacy_no_magic() { + // Random bytes with no OXCPT prefix. + let raw = b"some legacy bytes not starting with the magic"; + assert_eq!(BlobFormat::classify(raw), BlobFormat::Legacy); + } + + #[test] + fn classify_treats_short_magic_only_blob_as_legacy() { + // 5 bytes = magic only, no room for version+key_fp. Malformed + // v1; treated as Legacy so the decision tree flags it for + // rewrite instead of pretending it's a real v1 blob. + let raw = Vec::from(*OXCPT_MAGIC); + assert_eq!(BlobFormat::classify(&raw), BlobFormat::Legacy); + } + + #[test] + fn classify_empty_is_legacy() { + assert_eq!(BlobFormat::classify(&[]), BlobFormat::Legacy); + } + + #[tokio::test] + async fn head_format_matches_encrypted_head_pair_fp() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + let key = [0x77u8; 32]; + let backend = EncryptedBlobBackend::new_single_aes(local, &key); + match backend.head_format() { + BlobFormat::EncryptedV1 { key_fp } => { + assert_eq!(key_fp, KeyPair::new_aes_gcm(key).key_fp()); + } + other => panic!("expected EncryptedV1, got {other:?}"), + } + } + + #[tokio::test] + async fn head_format_is_plaintext_v1_for_none_head() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + // Empty pair list → wrapper synthesises a single `none:` pair. + let backend = EncryptedBlobBackend::new(local, vec![]); + assert_eq!(backend.head_format(), BlobFormat::PlaintextV1); + } + + #[tokio::test] + async fn read_and_classify_returns_plaintext_and_current_format() { + let tmp = TempDir::new().unwrap(); + let local = Arc::new(LocalBlobBackend::new(&tmp.path().join("blobs"))); + local.initialize().await.unwrap(); + + let k_old = [0xAAu8; 32]; + let k_new = [0xBBu8; 32]; + let hash = "7777777777777777777777777777777777777777777777777777777777777777"; + + // Write under the OLD key. + let writer = EncryptedBlobBackend::new_single_aes(local.clone(), &k_old); + writer + .put_blob_from_bytes(hash, Bytes::from_static(b"secret payload")) + .await + .unwrap(); + + // Reader has BOTH keys, k_new at head. Rotate scenario: + // classifier should report EncryptedV1{k_old_fp}, decrypt + // succeeds via key_fp lookup, plaintext round-trips. + let reader = EncryptedBlobBackend::new( + local, + vec![KeyPair::new_aes_gcm(k_old), KeyPair::new_aes_gcm(k_new)], + ); + let (plaintext, current) = reader.read_and_classify(hash).await.unwrap(); + assert_eq!(plaintext, b"secret payload".as_slice()); + match current { + BlobFormat::EncryptedV1 { key_fp } => { + assert_eq!(key_fp, KeyPair::new_aes_gcm(k_old).key_fp()); + } + other => panic!("expected EncryptedV1 with old fp, got {other:?}"), + } + // Confirms the rotate decision: current != head_format → + // rewrite (key rotation case). + assert_ne!(current, reader.head_format()); + } } From a9d5aae78114a322ef314bdd37ff326cfe838ea6 Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sat, 1 Aug 2026 23:46:46 +0200 Subject: [PATCH 06/29] feat(storage key rot): add rotate services --- .../services/storage_settings_service.rs | 15 +- src/common/di.rs | 29 ++ src/infrastructure/services/entry_backend.rs | 71 ++- src/infrastructure/services/mod.rs | 1 + .../services/storage_rotate_service.rs | 489 ++++++++++++++++++ src/interfaces/api/handlers/admin_handler.rs | 120 +++++ src/interfaces/middleware/server_status.rs | 119 +++-- tests/api/admin_jobs.hurl | 8 +- tests/api/storage_multi_entry.hurl | 8 +- 9 files changed, 786 insertions(+), 74 deletions(-) create mode 100644 src/infrastructure/services/storage_rotate_service.rs diff --git a/src/application/services/storage_settings_service.rs b/src/application/services/storage_settings_service.rs index 672df02b..6efac500 100644 --- a/src/application/services/storage_settings_service.rs +++ b/src/application/services/storage_settings_service.rs @@ -400,14 +400,23 @@ impl StorageSettingsService { entry, std::path::Path::new(&self.env_storage_config.root_dir), ); - let backend_kind = backend.backend_type().to_string(); + // Post-K2 (always-wrap): `backend.backend_type()` returns + // the outer wrapper's kind ("v1-plaintext" / "encrypted"), + // NOT the underlying storage backend. `health_check()` + // formats the wrapper-inner combo as + // `"()"` — that's the more informative + // string for the admin panel's Test-connection result. + // We use the wrapper-only string as a fallback for the + // health-check-failed branch, where there's no formatted + // status to draw from. + let fallback_backend_kind = backend.backend_type().to_string(); let status = match backend.health_check().await { Ok(s) => s, Err(e) => { return Ok(StorageTestResultDto { connected: false, message: format!("health-check failed: {e}"), - backend_type: backend_kind, + backend_type: fallback_backend_kind, available_bytes: None, roundtrip_passed: None, phase_reached: None, @@ -421,7 +430,7 @@ impl StorageSettingsService { let mut out = StorageTestResultDto { connected: status.connected, message: status.message, - backend_type: backend_kind, + backend_type: status.backend_type, available_bytes: status.available_bytes, roundtrip_passed: None, phase_reached: None, diff --git a/src/common/di.rs b/src/common/di.rs index 8a3e253e..6197fd94 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -2029,6 +2029,7 @@ impl AppServiceFactory { authorization: authorization.clone(), migration_readonly: migration_readonly.clone(), migration_progress: Arc::new(std::sync::RwLock::new(None)), + rotation_progress: Arc::new(std::sync::RwLock::new(None)), drive_repo: drive_repo.clone(), drive_management_service: Arc::new( crate::application::services::drive_management_service::DriveManagementService::new( @@ -2261,6 +2262,25 @@ impl AppServiceFactory { .register_recoverable_job(&app_state.core.job_registry, &job_store_provider_dyn) .await; + // K3: `storage_rotate` recoverable-job tenant. Same + // pattern as `storage_migration` but without the + // cutover/readonly plumbing — rotation writes in place on + // whichever entry the trigger endpoint names. Target name + // comes from `params.target_name` per run. + let _ = Arc::new( + crate::infrastructure::services::storage_rotate_service::StorageRotateService::new( + app_state + .maintenance_pool + .clone() + .expect("maintenance_pool set above"), + app_state.core.config.storage_entries.clone(), + self.storage_path.clone(), + app_state.rotation_progress.clone(), + ), + ) + .register_recoverable_job(&app_state.core.job_registry, &job_store_provider_dyn) + .await; + // 9b-2. Log whether system needs first-time admin setup if !admin_svc.is_system_initialized().await { tracing::warn!("╔══════════════════════════════════════════════════════════╗"); @@ -2795,6 +2815,15 @@ pub struct AppState { /// user's session banner about maintenance progress without /// polling. See `MigrationProgress` for the field shape. pub migration_progress: Arc>>, + /// Live progress snapshot for the storage-rotate handler + /// (`storage_rotate` — K3 of the storage-key-rotation plan). + /// `Some(_)` while a rotation is running; `None` otherwise. + /// Held separately from `migration_progress` so the + /// server-status header can broadcast the two states + /// independently: migration engages readonly mode, rotation + /// does not. Same `MigrationProgress` type — both are "walk + /// progress" fundamentally. + pub rotation_progress: Arc>>, /// Drive entity repository — `GET /api/drives`, the personal-drive /// lifecycle hook, and (post-D2) shared-drive creation flow all read /// through this. Backing table is `storage.drives`; membership is diff --git a/src/infrastructure/services/entry_backend.rs b/src/infrastructure/services/entry_backend.rs index 932947d6..0832e2d5 100644 --- a/src/infrastructure/services/entry_backend.rs +++ b/src/infrastructure/services/entry_backend.rs @@ -194,11 +194,47 @@ pub async fn resolve_active_entry<'a>( /// /// Both are boot-fatal and indicate a code (not config) bug, so /// panic is the honest response. -pub fn build_entry_backend( +/// Typed variant of [`build_entry_backend`] — returns the concrete +/// [`EncryptedBlobBackend`] wrapper so callers that need K3's +/// introspection API (`read_and_classify`, `head_format`, …) can hit +/// it directly without a downcast. +/// +/// Same construction path as `build_entry_backend`; the trait-object +/// version delegates through this. Preferred for job handlers +/// (`storage_rotate`) that need typed access. The trait-object +/// version stays for the DI hot-path where the caller only needs +/// the generic `BlobStorageBackend` contract. +pub fn build_entry_backend_typed( + entry: &NamedStorageEntry, + local_storage_path_fallback: &Path, +) -> Arc { + let base = build_base_backend(entry, local_storage_path_fallback); + let pairs = entry.encryption.clone().unwrap_or_default(); + let mode = match entry.head_cipher() { + Some(crate::common::config::CipherKind::AesGcm256) => "encrypted-v1", + _ => "plaintext-v1", + }; + tracing::info!( + "Storage entry `{}` — {} wrapper (pairs: {})", + entry.name, + mode, + pairs.len() + ); + Arc::new( + crate::infrastructure::services::encrypted_blob_backend::EncryptedBlobBackend::new( + base, pairs, + ), + ) +} + +/// Construct just the raw backend for the entry (no wrapper). Split +/// out of [`build_entry_backend`] so the typed variant can share the +/// switch on backend type without duplicating panic messages. +fn build_base_backend( entry: &NamedStorageEntry, local_storage_path_fallback: &Path, ) -> Arc { - let base: Arc = match entry.backend { + match entry.backend { StorageBackendType::Local => { let path = entry .root_dir @@ -227,27 +263,12 @@ pub fn build_entry_backend( }); Arc::new(crate::infrastructure::services::azure_blob_backend::AzureBlobBackend::new(az)) } - }; - - // v1 wrapper (Choice 1/B: always wrap). Every entry gets the - // header-aware read/write path — even entries with no - // `_ENCRYPTION_KEY` at all. This normalises the on-disk format - // going forward: all new writes carry the OXCPT v1 header, all - // reads magic-byte-dispatch (with legacy fallback for - // header-less pre-K2 blobs). Not backwards-compatible with - // pre-K2 code trying to read new writes — but Ed's called it: - // uniform format is worth the one-way door. - use crate::infrastructure::services::encrypted_blob_backend::EncryptedBlobBackend; - let pairs = entry.encryption.clone().unwrap_or_default(); - let mode = match entry.head_cipher() { - Some(crate::common::config::CipherKind::AesGcm256) => "encrypted-v1", - _ => "plaintext-v1", - }; - tracing::info!( - "Storage entry `{}` — {} wrapper (pairs: {})", - entry.name, - mode, - pairs.len() - ); - Arc::new(EncryptedBlobBackend::new(base, pairs)) + } +} + +pub fn build_entry_backend( + entry: &NamedStorageEntry, + local_storage_path_fallback: &Path, +) -> Arc { + build_entry_backend_typed(entry, local_storage_path_fallback) } diff --git a/src/infrastructure/services/mod.rs b/src/infrastructure/services/mod.rs index b7ab9895..cc6261bf 100644 --- a/src/infrastructure/services/mod.rs +++ b/src/infrastructure/services/mod.rs @@ -46,6 +46,7 @@ pub mod search_index; pub mod share_unlock_cookie; pub mod smtp_email_sender; pub mod storage_migration_service; +pub mod storage_rotate_service; pub mod swappable_blob_backend; pub mod thumbnail_service; #[cfg(test)] diff --git a/src/infrastructure/services/storage_rotate_service.rs b/src/infrastructure/services/storage_rotate_service.rs new file mode 100644 index 00000000..bf66ad7b --- /dev/null +++ b/src/infrastructure/services/storage_rotate_service.rs @@ -0,0 +1,489 @@ +//! Storage-format rotation as a recoverable-run tenant (K3 of +//! `docs/plan/storage-key-rotation.md`). +//! +//! Iterates `storage.blobs` for a target entry, decides per blob +//! whether the on-disk format matches what the entry's head pair +//! would write, and rewrites in place when it doesn't. Covers four +//! transitions with a single equality check: +//! +//! * Legacy blob (no `OXCPT` magic) → rewrite as v1 with the head +//! pair's format. +//! * v1 encrypted, decrypted under a pair-index other than head → +//! rewrite (key rotation). +//! * v1 plaintext with head=`aes:K` → rewrite (encrypt-in-place). +//! * v1 encrypted with head=`none:` → rewrite (decrypt-in-place). +//! +//! ### No readonly, no cutover +//! +//! `storage_rotate` is per-blob idempotent — repeat rewrites are +//! byte-safe (content-addressability holds; the wrapper always +//! produces the head format). Concurrent user writes coexist: they +//! land as head-format themselves, so when the walk reaches that +//! hash the classifier reports "already at head format" and the +//! decision tree collapses to `skip`. No app-wide read-only gate is +//! ever engaged — a critical improvement over `storage_migration`, +//! whose target-different-from-source cutover forces one. +//! +//! ### Restart survival +//! +//! Cursor + per-blob failure findings are persisted after every +//! batch. On restart, boot flips any abandoned `Running` row to +//! `Paused`; an admin trigger resumes from the checkpointed cursor. +//! The last checkpoint window (~100 blobs) re-processes; each of +//! those blobs is now head-format from the previous run's rewrite, +//! so the walk short-circuits without re-writing. Effectively free. +//! +//! ### Design notes +//! +//! * **Cursor** — UTF-8 hex of the last-processed blob hash (64 +//! chars). Same encoding as `storage_migration` and +//! `blobs_consistency`. +//! * **Target lookup** — the entry NAME is stashed in `params` at +//! Fresh-open time and re-read on Resume. The wrapper for that +//! entry is rebuilt at the top of every run via +//! `build_entry_backend_typed`; mid-run config changes are +//! ignored until the next run (mirrors `storage_migration`). +//! * **Per-blob failures don't fail the run** — each failure records +//! a `rotation_failed` finding (severity `data_loss` — the bytes +//! didn't get rewritten) and the walk continues. A run that +//! completes with zero findings is proof every blob is at head +//! format. +//! * **`?deep=true` is unused** — rotation has no slow variant. +//! Parameter accepted for uniformity with other tenants; ignored. + +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use sqlx::PgPool; + +use crate::application::ports::blob_storage_ports::BlobStorageBackend; +use crate::common::config::NamedStorageEntry; +use crate::common::migration_progress::MigrationProgress; +use crate::infrastructure::scheduler::{ + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome, + RunStatus, record_or_log, +}; +use crate::infrastructure::services::entry_backend::build_entry_backend_typed; + +pub const STORAGE_ROTATE_JOB_NAME: &str = "storage_rotate"; + +/// The `params` JSONB key under which the run's target entry name is +/// stashed at Fresh-open time via `JobStore::set_string_param`. +/// Kept identical to `storage_migration`'s TARGET_NAME_PARAM so +/// operators grepping run rows see the same convention across both +/// storage-touching tenants. +pub const TARGET_NAME_PARAM: &str = "target_name"; + +/// Rows per batch. Matches `storage_migration` / `blobs_consistency` +/// so the checkpoint + cancel-poll cadence is uniform across tenants. +const BATCH_SIZE: i64 = 100; + +pub struct StorageRotateService { + pool: Arc, + /// Immutable per-deploy snapshot; used to look up the target + /// entry by name at run start. Matches `AppConfig.storage_entries`. + storage_entries: Vec, + /// Ambient `AppConfig.storage_path` used as the `root_dir` + /// fallback for a Local target entry that doesn't declare its + /// own `_ROOT_DIR`. Same fallback rule as boot + /// (`build_entry_backend`). + storage_path_fallback: PathBuf, + /// Shared in-memory progress snapshot for the server-status + /// header middleware. `Some(_)` while a rotation is + /// running/paused, `None` otherwise. Distinct from + /// `AppState.migration_progress` so the header can broadcast + /// migration + rotation states independently. + rotation_progress: Arc>>, +} + +impl StorageRotateService { + pub fn new( + pool: Arc, + storage_entries: Vec, + storage_path_fallback: PathBuf, + rotation_progress: Arc>>, + ) -> Self { + Self { + pool, + storage_entries, + storage_path_fallback, + rotation_progress, + } + } + + /// Chainable self-registration — mirrors the `*_consistency` + /// tenants and `storage_migration`. On-demand only (no periodic + /// tick). + pub async fn register_recoverable_job( + self: Arc, + registry: &JobRegistry, + provider: &Arc, + ) -> Arc { + registry + .register_recoverable_job(self.clone(), provider.clone(), None) + .await; + self + } +} + +#[async_trait] +impl RecoverableJobHandler for StorageRotateService { + fn name(&self) -> &str { + STORAGE_ROTATE_JOB_NAME + } + + /// Definitive count — one row per blob. Same query as + /// `storage_migration::count_total`; the two walk the same rows. + async fn count_total(&self) -> Option { + let row: Result<(i64,), sqlx::Error> = sqlx::query_as("SELECT COUNT(*) FROM storage.blobs") + .fetch_one(self.pool.as_ref()) + .await; + match row { + Ok((n,)) => Some(n.max(0) as u64), + Err(e) => { + tracing::debug!( + target: "oxicloud::rotate", + event = "storage_rotate.count_total_failed", + error = %e, + "count_total failed — run will not surface a progress bar" + ); + None + } + } + } + + async fn run_resumable( + &self, + store: &dyn JobStore, + args: &JobRunArgs, + resume_cursor: Option>, + ) -> RunOutcome { + // Resolve target entry name — same shape as `storage_migration`. + let is_fresh = resume_cursor.is_none(); + let target_name = if is_fresh { + let Some(name) = args.storage.clone() else { + return RunOutcome::Failed { + message: "storage_rotate requires `target_name` on a fresh run — trigger via \ + POST /api/admin/storage/entries/{name}/rotate" + .to_string(), + }; + }; + if let Err(e) = store.set_string_param(TARGET_NAME_PARAM, &name).await { + return RunOutcome::Failed { + message: format!("failed to persist target_name to params: {e}"), + }; + } + name + } else { + match store.get_string_param(TARGET_NAME_PARAM).await { + Ok(Some(name)) => name, + Ok(None) => { + return RunOutcome::Failed { + message: format!( + "resumed run has no {TARGET_NAME_PARAM} in params — cancel + trigger \ + fresh." + ), + }; + } + Err(e) => { + return RunOutcome::Failed { + message: format!("read {TARGET_NAME_PARAM} from params: {e}"), + }; + } + } + }; + + // Look up the target entry. + let target_entry = match self.storage_entries.iter().find(|e| e.name == target_name) { + Some(e) => e, + None => { + let available = if self.storage_entries.is_empty() { + "(none)".to_string() + } else { + self.storage_entries + .iter() + .map(|e| e.name.as_str()) + .collect::>() + .join(", ") + }; + return RunOutcome::Failed { + message: format!( + "target entry `{target_name}` not declared in `OXICLOUD_STORAGE_ENTRIES` — \ + available: {available}." + ), + }; + } + }; + + // Build the wrapper for this entry — typed so we can call + // `read_and_classify` + `head_format` directly. + let wrapper = build_entry_backend_typed(target_entry, &self.storage_path_fallback); + if let Err(e) = wrapper.initialize().await { + return RunOutcome::Failed { + message: format!("target entry `{target_name}` failed to initialize: {e}"), + }; + } + let head_format = wrapper.head_format(); + + tracing::info!( + target: "audit", + event = "storage_rotate.run_started", + run_id = %store.run_id(), + target_name = %target_name, + head_format = ?head_format, + resuming = !is_fresh, + "storage_rotate started on `{target_name}` (head_format = {head_format:?})" + ); + + // Seed the progress snapshot. Total = count_total's estimate; + // if that failed we still surface the header without a + // denominator so the banner shows "rotation in progress" at + // minimum. + let total = self.count_total().await.unwrap_or(0); + { + let mut guard = self + .rotation_progress + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *guard = Some(MigrationProgress::new(target_name.clone(), total)); + } + + let mut cursor: Option = match resume_cursor { + None => None, + Some(bytes) if bytes.is_empty() => None, + Some(bytes) => match String::from_utf8(bytes) { + Ok(s) => Some(s), + Err(e) => { + self.clear_progress(); + return RunOutcome::Failed { + message: format!("invalid cursor: not valid UTF-8: {e}"), + }; + } + }, + }; + + let mut rewritten_count = 0u64; + let mut skipped_count = 0u64; + let mut failed_count = 0u64; + + loop { + // Cooperative cancel poll between batches. + match store.status().await { + Ok(RunStatus::CancelRequested) => { + self.clear_progress(); + tracing::info!( + target: "oxicloud::rotate", + event = "storage_rotate.cancelled", + run_id = %store.run_id(), + rewritten = rewritten_count, + skipped = skipped_count, + failed = failed_count, + "storage_rotate cancelled cooperatively, pausing" + ); + return RunOutcome::Paused { + cursor: cursor + .as_ref() + .map(|s| s.as_bytes().to_vec()) + .unwrap_or_default(), + }; + } + Ok(_) => {} + Err(e) => { + self.clear_progress(); + return RunOutcome::Failed { + message: format!("status poll: {e}"), + }; + } + } + + // Fetch the next batch. Same keyset pagination shape as + // `storage_migration` — `hash > $1` on the PK, index-only. + let rows: Vec<(String,)> = match sqlx::query_as( + r#" + SELECT hash + FROM storage.blobs + WHERE ($1::text IS NULL OR hash > $1) + ORDER BY hash + LIMIT $2 + "#, + ) + .bind(cursor.as_deref()) + .bind(BATCH_SIZE) + .fetch_all(self.pool.as_ref()) + .await + { + Ok(r) => r, + Err(e) => { + self.clear_progress(); + return RunOutcome::Failed { + message: format!("batch fetch: {e}"), + }; + } + }; + + if rows.is_empty() { + return self + .finish_completed( + store, + &target_name, + rewritten_count, + skipped_count, + failed_count, + ) + .await; + } + + for (hash,) in &rows { + // Read + classify in one round-trip. Failure here is + // a real read failure (e.g. blob missing on disk), + // recorded as a finding. + let (plaintext, current_format) = match wrapper.read_and_classify(hash).await { + Ok(pair) => pair, + Err(e) => { + failed_count += 1; + tracing::warn!( + target: "oxicloud::rotate", + event = "storage_rotate.read_failed", + run_id = %store.run_id(), + hash = %hash, + error = %e, + "failed to read blob for classification; recording finding" + ); + record_or_log( + store, + STORAGE_ROTATE_JOB_NAME, + "rotation_failed", + "data_loss", + None, + serde_json::json!({ + "hash": hash, + "phase": "read", + "error": e.to_string(), + }), + ) + .await; + continue; + } + }; + + // The whole decision tree collapses to one equality + // check thanks to `BlobFormat`'s `PartialEq`. Six + // cases in the plan → one branch here. + if current_format == head_format { + skipped_count += 1; + continue; + } + + // Rewrite via the standard write path — atomic + // replace at the same object key. `put_blob_from_bytes` + // frames the plaintext with the head pair's format + // (encrypted-v1 or plaintext-v1) and hands the + // resulting bytes to the inner backend. + if let Err(e) = wrapper + .put_blob_from_bytes(hash, Bytes::from(plaintext.to_vec())) + .await + { + failed_count += 1; + tracing::warn!( + target: "oxicloud::rotate", + event = "storage_rotate.write_failed", + run_id = %store.run_id(), + hash = %hash, + error = %e, + "failed to rewrite blob; recording finding" + ); + record_or_log( + store, + STORAGE_ROTATE_JOB_NAME, + "rotation_failed", + "data_loss", + None, + serde_json::json!({ + "hash": hash, + "phase": "write", + "from": format!("{current_format:?}"), + "to": format!("{head_format:?}"), + "error": e.to_string(), + }), + ) + .await; + continue; + } + rewritten_count += 1; + } + + // Advance cursor + checkpoint. `delta_count` = work + // attempted this batch, so the progress bar advances even + // when a batch is dominated by skips (steady-state + // re-run) or failures. + let last_hash = rows.last().map(|(h,)| h.clone()).expect("non-empty rows"); + cursor = Some(last_hash.clone()); + let batch_len = rows.len() as u64; + if let Err(e) = store.checkpoint(last_hash.into_bytes(), batch_len).await { + self.clear_progress(); + return RunOutcome::Failed { + message: format!("checkpoint: {e}"), + }; + } + { + let mut guard = self + .rotation_progress + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(progress) = guard.as_mut() { + progress.bump(batch_len); + } + } + + if (rows.len() as i64) < BATCH_SIZE { + return self + .finish_completed( + store, + &target_name, + rewritten_count, + skipped_count, + failed_count, + ) + .await; + } + } + } +} + +impl StorageRotateService { + /// Terminal successful path — clear the header snapshot and log a + /// final audit line. Unlike `storage_migration::finish_completed` + /// there's no cutover / hot-swap step: rotation writes in place + /// on the entry that's already there. + async fn finish_completed( + &self, + store: &dyn JobStore, + target_name: &str, + rewritten: u64, + skipped: u64, + failed: u64, + ) -> RunOutcome { + self.clear_progress(); + tracing::info!( + target: "audit", + event = "storage_rotate.run_completed", + run_id = %store.run_id(), + target_name = %target_name, + rewritten = rewritten, + skipped = skipped, + failed = failed, + "storage_rotate completed on `{target_name}` — {rewritten} rewritten, {skipped} skipped, {failed} failed" + ); + RunOutcome::Completed + } + + fn clear_progress(&self) { + let mut guard = self + .rotation_progress + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *guard = None; + } +} diff --git a/src/interfaces/api/handlers/admin_handler.rs b/src/interfaces/api/handlers/admin_handler.rs index c000e520..433e3194 100644 --- a/src/interfaces/api/handlers/admin_handler.rs +++ b/src/interfaces/api/handlers/admin_handler.rs @@ -85,6 +85,14 @@ pub fn admin_routes() -> Router> { .route("/storage/migration/start", post(start_migration)) .route("/storage/migration/pause", post(pause_migration)) .route("/storage/migration/resume", post(resume_migration)) + // K3 (storage-key-rotation): per-entry rotate trigger. + // Normalises every blob on the named entry to its head-pair + // format (legacy → v1, plaintext ↔ encrypted, old-key → + // new-key). No readonly mode; safe under normal traffic. + .route( + "/storage/entries/{name}/rotate", + post(trigger_storage_rotate), + ) // NOTE: /storage/migration/verify retired in slice 7 (see the // comment near where `verify_migration` used to live). Use // `POST /api/admin/jobs/blobs_consistency/trigger?storage=`. @@ -621,6 +629,118 @@ async fn trigger_storage_migration( .into_response()) } +/// POST /api/admin/storage/entries/{name}/rotate — trigger the +/// `storage_rotate` recoverable job on a specific entry. +/// +/// Normalises every blob on `` to the entry's head-pair +/// format: legacy → v1, plaintext ↔ encrypted, old-key → new-key. +/// See `docs/plan/storage-key-rotation.md` §"The rotation job". +/// +/// Unlike migration, rotation does NOT engage read-only mode — +/// rewrites happen in place under normal traffic. Concurrent user +/// writes coexist safely. +/// +/// The handler validates the entry name synchronously (400 on +/// unknown entry); the actual walk detaches into a +/// `tokio::spawn` so the HTTP call returns immediately. +#[utoipa::path( + post, + path = "/api/admin/storage/entries/{name}/rotate", + responses( + (status = 202, description = "Rotation dispatched"), + (status = 400, description = "Unknown entry"), + (status = 401, description = "Unauthorized"), + (status = 403, description = "Admin required") + ), + params( + ("name" = String, Path, description = "Storage entry name to rotate") + ), + security(("bearerAuth" = [])), + tag = "admin" +)] +pub async fn trigger_storage_rotate( + State(state): State>, + axum::extract::Path(name): axum::extract::Path, +) -> Result { + use crate::infrastructure::scheduler::JobRunArgs; + use crate::infrastructure::services::storage_migration_service::STORAGE_MIGRATION_JOB_NAME; + use crate::infrastructure::services::storage_rotate_service::STORAGE_ROTATE_JOB_NAME; + + // Synchronous entry-existence check — a bad name would fail the + // run anyway, but returning 400 here spares the operator an + // audit-log round-trip. + let entries = &state.core.config.storage_entries; + if entries.iter().all(|e| e.name != name) { + let available = if entries.is_empty() { + "(none)".to_string() + } else { + entries + .iter() + .map(|e| e.name.as_str()) + .collect::>() + .join(", ") + }; + return Err(AppError::bad_request(format!( + "unknown storage entry `{name}`. Available: [{available}]" + ))); + } + + // Concurrency guard per plan: at most one encryption-touching + // recoverable run at a time across the whole app. Rotation + // rewrites blobs in place; migration copies + swaps; running + // both simultaneously could interleave writes on the same + // hash. Cheap check — `list_runs` limit 1 with the status + // filter is an index scan. + let provider = state.core.job_store_provider.clone(); + for job_name in [STORAGE_ROTATE_JOB_NAME, STORAGE_MIGRATION_JOB_NAME] { + let in_flight = provider + .list_runs(job_name, 5) + .await + .map_err(AppError::from)? + .into_iter() + .any(|r| { + matches!( + r.status, + crate::infrastructure::scheduler::RunStatus::Running + | crate::infrastructure::scheduler::RunStatus::Paused + | crate::infrastructure::scheduler::RunStatus::CancelRequested + ) + }); + if in_flight { + return Err(AppError::bad_request(format!( + "cannot start storage_rotate on `{name}` — `{job_name}` is already Running / \ + Paused / CancelRequested. Wait for it to finish (or cancel via \ + `POST /api/admin/jobs/{job_name}/cancel`)." + ))); + } + } + + tracing::info!( + target: "audit", + event = "storage_rotate.trigger_requested", + target_name = %name, + "👮🏻‍♂️ Admin triggered storage_rotate on `{name}`" + ); + + let registry = state.core.job_registry.clone(); + let args = JobRunArgs { + storage: Some(name.clone()), + ..JobRunArgs::default() + }; + tokio::spawn(async move { + registry.trigger(STORAGE_ROTATE_JOB_NAME, &args).await; + }); + + Ok(( + StatusCode::ACCEPTED, + Json(serde_json::json!({ + "message": format!("Rotation dispatched on `{name}` — poll GET /api/admin/jobs/{STORAGE_ROTATE_JOB_NAME} for progress"), + "detached": true, + })), + ) + .into_response()) +} + /// Idle-state DTO — no run has been triggered yet. fn idle_migration_dto() -> MigrationStateDto { MigrationStateDto { diff --git a/src/interfaces/middleware/server_status.rs b/src/interfaces/middleware/server_status.rs index 2df6c57a..80926376 100644 --- a/src/interfaces/middleware/server_status.rs +++ b/src/interfaces/middleware/server_status.rs @@ -9,23 +9,22 @@ //! //! ## Cost model //! -//! On the *hot path* (no migration running — the ~100% case in normal -//! operation) this middleware does: +//! On the *hot path* (no migration AND no rotation running — the +//! ~100% case in normal operation) this middleware does: //! 1. one `AtomicBool::load(Relaxed)` — sub-nanosecond; -//! 2. an early return when `false`. +//! 2. one `RwLock::read` on `rotation_progress` — uncontended; +//! 3. an early return when both are inactive. //! -//! No allocation, no lock, no formatting. Adds no measurable latency -//! at any user count. +//! No allocation, no formatting on the hot path. The rotation-check +//! `RwLock::read` is cheap because writers only fire on batch +//! checkpoints (~every 100 blobs); worst-case contention is +//! sub-microsecond. //! -//! On the *cold path* (migration in progress) this middleware does: -//! 1. the atomic load above; -//! 2. one `RwLock::read` (uncontended — writers are the migration -//! handler, one per batch every ~100 blobs); -//! 3. one small `serde_json::to_string` call on a 4-field struct -//! (a few dozen bytes); -//! 4. one header insertion. +//! On the *cold path* (migration OR rotation in progress) the +//! payload builder pulls the progress snapshot(s), formats a small +//! JSON struct (~a few dozen bytes) and inserts the header. //! -//! Total per-request work in this branch: microseconds. +//! Total per-request work on cold path: microseconds. use axum::extract::Request; use axum::extract::State; @@ -53,11 +52,20 @@ pub const SERVER_STATUS_HEADER: &str = "x-server-status"; struct HeaderPayload { readonly: bool, #[serde(skip_serializing_if = "Option::is_none")] - migration: Option, + migration: Option, + /// K3: independent of `readonly` — rotation does NOT engage the + /// app-wide read-only flag, so the frontend needs a distinct + /// signal to know "rotation is running, show the rotation + /// banner instead of migration banner". + #[serde(skip_serializing_if = "Option::is_none")] + rotation: Option, } +/// Shared progress shape used by both `migration` and `rotation` +/// header fields — same struct name, same JSON field names. Frontend +/// treats them identically at the render layer. #[derive(serde::Serialize)] -struct MigrationHeader { +struct ProgressHeader { // `target` is owned here — the RwLock guard is released before // serialisation, so a borrowed slice wouldn't survive. Names // are small (`[a-z0-9_-]{1,32}`) so the copy is trivial. @@ -67,49 +75,76 @@ struct MigrationHeader { percent: u8, } +impl ProgressHeader { + fn from_snapshot(p: &crate::common::migration_progress::MigrationProgress) -> Self { + Self { + target: p.target_name.clone(), + migrated: p.migrated_blobs, + total: p.total_blobs, + percent: p.percent, + } + } +} + pub async fn server_status_middleware( State(state): State>, request: Request, next: Next, ) -> Response { - // Hot-path fast return. When no migration is running the flag is - // false and there's nothing to emit — a bare atomic load and out. let readonly = state.migration_readonly.load(Ordering::Relaxed); + + // Rotation snapshot check — cheap uncontended `read`; if `None` + // and readonly is also false, hot-path returns without a header. + let rotation_active = state + .rotation_progress + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_some(); + let mut response = next.run(request).await; - if !readonly { + if !readonly && !rotation_active { return response; } - // Cold path — build the payload from the shared progress - // snapshot. If the snapshot is absent (readonly is true but the - // handler hasn't seeded progress yet, or a restart-during- - // migration scenario) we still emit `readonly: true` so the - // banner shows — the frontend renders a "maintenance in progress" - // message even when specific numbers aren't available. + // Cold path — build the payload from whichever snapshots are + // active. `readonly:true` fires the migration banner even if + // the migration handler hasn't seeded its progress yet + // (restart-mid-migration scenario). `rotation` is populated + // independently. let payload = { - let guard = state - .migration_progress - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner); + let migration = if readonly { + state + .migration_progress + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .map(ProgressHeader::from_snapshot) + } else { + None + }; + let rotation = if rotation_active { + state + .rotation_progress + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .map(ProgressHeader::from_snapshot) + } else { + None + }; HeaderPayload { - readonly: true, - migration: guard.as_ref().map(|p| MigrationHeader { - target: p.target_name.clone(), - migrated: p.migrated_blobs, - total: p.total_blobs, - percent: p.percent, - }), + readonly, + migration, + rotation, } }; - // `serde_json::to_string` on this 4-field struct is a few - // dozen-byte allocation — negligible against the response body. - // A serialize failure here would be a programming bug (all - // fields are trivially serializable), so we degrade to a - // minimal `readonly: true` string rather than skipping the - // header entirely. + // `serde_json::to_string` on this struct is a few dozen-byte + // allocation — negligible against the response body. A + // serialize failure here would be a programming bug, so we + // degrade to a minimal string rather than skipping the header. let value = - serde_json::to_string(&payload).unwrap_or_else(|_| r#"{"readonly":true}"#.to_string()); + serde_json::to_string(&payload).unwrap_or_else(|_| r#"{"readonly":false}"#.to_string()); if let Ok(header_value) = HeaderValue::from_str(&value) { response .headers_mut() diff --git a/tests/api/admin_jobs.hurl b/tests/api/admin_jobs.hurl index 35991bc3..78f7a850 100644 --- a/tests/api/admin_jobs.hurl +++ b/tests/api/admin_jobs.hurl @@ -96,15 +96,17 @@ jsonpath "$..interval_ms" count == 3 # wrapped by RecoverableAdapter so they appear here alongside the # periodics) + 1 coordinator (consistency_batch — a plain # JobHandler that dispatches every registered `*_consistency`) + -# 1 on-demand admin op (storage_migration — recoverable, no -# periodic tick, triggered by the admin panel's backend cutover). +# 2 on-demand admin ops (storage_migration — the readonly-mode + +# cutover backend swap; storage_rotate — K3, in-place per-blob +# format normalisation, no readonly). # Bump when a new tenant registers. -jsonpath "$..running" count == 11 +jsonpath "$..running" count == 12 jsonpath "$[*].name" contains "drives_consistency" jsonpath "$[*].name" contains "folders_consistency" jsonpath "$[*].name" contains "files_consistency" jsonpath "$[*].name" contains "consistency_batch" jsonpath "$[*].name" contains "storage_migration" +jsonpath "$[*].name" contains "storage_rotate" # ───────────────────────────────────────────────────────────── diff --git a/tests/api/storage_multi_entry.hurl b/tests/api/storage_multi_entry.hurl index d12c0d23..49bfe561 100644 --- a/tests/api/storage_multi_entry.hurl +++ b/tests/api/storage_multi_entry.hurl @@ -87,7 +87,13 @@ Content-Type: application/json HTTP 200 [Asserts] jsonpath "$.connected" == true -jsonpath "$.backend_type" == "local" +# Post-K2 (storage-key-rotation): every entry is wrapped in the v1 +# blob-format decorator, so `backend_type` reports the WRAPPER's kind +# in `"()"` form. `local_main` is unencrypted → wrapper +# is `v1-plaintext`. Match on the inner name via `contains` so the +# assertion survives future wrapper renames. +jsonpath "$.backend_type" contains "local" +jsonpath "$.backend_type" contains "v1-plaintext" jsonpath "$.roundtrip_passed" == true jsonpath "$.phase_reached" == "cleanup_ok" From a58351b7ad098a0769c4ec83b50a45229c332616 Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sun, 2 Aug 2026 00:06:08 +0200 Subject: [PATCH 07/29] feat(storage key rot): add admin panel --- frontend/src/lib/api/endpoints/admin.ts | 16 +++ .../src/lib/components/AdminJobsPanel.svelte | 8 +- frontend/src/lib/components/AppShell.svelte | 7 ++ .../src/lib/components/ReadOnlyBanner.svelte | 88 +++++++++----- .../src/lib/stores/serverStatus.svelte.ts | 41 +++++-- .../src/routes/admin/[[tab]]/+page.svelte | 107 ++++++++++++++++++ frontend/static/locales/en.json | 4 +- frontend/static/locales/fr.json | 3 + src/infrastructure/scheduler/handler.rs | 16 +++ src/infrastructure/scheduler/recoverable.rs | 8 ++ src/infrastructure/scheduler/registry.rs | 6 + src/interfaces/api/handlers/admin_handler.rs | 21 ++++ 12 files changed, 282 insertions(+), 43 deletions(-) diff --git a/frontend/src/lib/api/endpoints/admin.ts b/frontend/src/lib/api/endpoints/admin.ts index a4001438..9f9da765 100644 --- a/frontend/src/lib/api/endpoints/admin.ts +++ b/frontend/src/lib/api/endpoints/admin.ts @@ -538,6 +538,22 @@ export function migrationAction( return mutate(`/api/admin/storage/migration/${action}`, 'POST', body); } +/** + * K4 (storage-key-rotation): trigger `storage_rotate` on a specific + * storage entry. Normalises every blob on `` to that entry's + * head-pair format: legacy → v1, plaintext ↔ encrypted, old-key → + * new-key. Fire-and-forget — poll `GET /api/admin/jobs/storage_rotate` + * for status. + * + * Backend: `POST /api/admin/storage/entries/{name}/rotate` + * (`admin_handler::trigger_storage_rotate`). Refuses (400) on unknown + * entry name or when a `storage_rotate` / `storage_migration` run is + * already in flight. + */ +export function rotateStorageEntry(name: string): Promise { + return mutate(`/api/admin/storage/entries/${encodeURIComponent(name)}/rotate`, 'POST', undefined); +} + // verifyMigration + MigrationVerifyResult retired in slice 7 of // docs/plan/storage-multi-entry.md — the corresponding backend // endpoint's sample-based check is superseded by diff --git a/frontend/src/lib/components/AdminJobsPanel.svelte b/frontend/src/lib/components/AdminJobsPanel.svelte index 0d55d949..f7140657 100644 --- a/frontend/src/lib/components/AdminJobsPanel.svelte +++ b/frontend/src/lib/components/AdminJobsPanel.svelte @@ -470,7 +470,13 @@ return name_is_recoverable(job.name); } function name_is_recoverable(name: string): boolean { - return name.endsWith('_consistency') || name === 'storage_migration'; + // K3 storage-key-rotation adds `storage_rotate` to the recoverable + // tenant set. Same shape as `storage_migration` — walks blobs, + // records findings, supports resume from cursor — so it needs the + // same expand/runs/findings surface. + return ( + name.endsWith('_consistency') || name === 'storage_migration' || name === 'storage_rotate' + ); } // Consistency batch shortcut — top button. Only shown when the diff --git a/frontend/src/lib/components/AppShell.svelte b/frontend/src/lib/components/AppShell.svelte index d669b4dd..5dc8f067 100644 --- a/frontend/src/lib/components/AppShell.svelte +++ b/frontend/src/lib/components/AppShell.svelte @@ -1041,6 +1041,13 @@ different copy. --> {#if serverStatus().readonly} + {:else if serverStatus().rotation} + + {/if} {@render children()} diff --git a/frontend/src/lib/components/ReadOnlyBanner.svelte b/frontend/src/lib/components/ReadOnlyBanner.svelte index d7fd5491..847ec41d 100644 --- a/frontend/src/lib/components/ReadOnlyBanner.svelte +++ b/frontend/src/lib/components/ReadOnlyBanner.svelte @@ -1,37 +1,27 @@