diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 8959cd65..e1700864 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -168,6 +168,7 @@ export default defineConfig({ { text: "ReBAC Authorization", link: "/architecture/rebac-authorization" }, { text: "Share Integration", link: "/architecture/share-integration" }, { text: "Storage Quotas", link: "/architecture/storage-quotas" }, + { text: "Backend Storage", link: "/architecture/backend-storage" }, { text: "File and Blob lifecycle", link: "/architecture/file-and-blob-lifecycle" }, { text: "ReBAC & Authorization", link: "/architecture/rebac-authorization" }, { text: "User lifecycle", link: "/architecture/user-lifecycle" }, diff --git a/docs/architecture/backend-storage.md b/docs/architecture/backend-storage.md new file mode 100644 index 00000000..cf22ff17 --- /dev/null +++ b/docs/architecture/backend-storage.md @@ -0,0 +1,421 @@ +# Backend Storage + +Reference for implementors of storage backends, encryption layers, +consistency checks, and migration/rotation jobs. + +The user-facing "how to configure S3" guide lives in +[`docs/config/env.md`](../config/env.md); the operational plan +history is in [`docs/plan/storage-multi-entry.md`](../plan/storage-multi-entry.md) +and [`docs/plan/storage-key-rotation.md`](../plan/storage-key-rotation.md). +This page is the "what the code actually does and why" reference. + +--- + +## Supported backends + +Three concrete implementations of `BlobStorageBackend` ship in the +tree today. All go through the same `EncryptedBlobBackend` wrapper +(see §6) so encryption, header format, BLAKE3 rescue, smart-skip +probe, and lifecycle behaviour are uniform. + +| Backend | `StorageBackendType` | Env key prefix | Config surface | Impl | +|---|---|---|---|---| +| **Local filesystem** | `Local` | `OXICLOUD_STORAGE__ROOT_DIR` | Root directory on the host FS. Shard tree `/.blobs//`. Atomic replace via tempfile + `rename(2)`. | [`local_blob_backend.rs`](../../src/infrastructure/services/local_blob_backend.rs) | +| **S3-compatible** | `S3` | `OXICLOUD_STORAGE__BUCKET` + `_REGION` + `_ACCESS_KEY` + `_SECRET_KEY` + optional `_ENDPOINT_URL` + `_FORCE_PATH_STYLE` | AWS S3, Cloudflare R2, Backblaze B2, MinIO, DigitalOcean Spaces, Wasabi — anything speaking the S3 API. | [`s3_blob_backend.rs`](../../src/infrastructure/services/s3_blob_backend.rs) | +| **Azure Blob Storage** | `Azure` | `OXICLOUD_STORAGE__ACCOUNT_NAME` + `_ACCOUNT_KEY` (or `_SAS_TOKEN`) + `_CONTAINER` + optional `_ENDPOINT_URL` | Azure Blob Storage; `_ENDPOINT_URL` targets Azurite (local emulator) or private endpoints. | [`azure_blob_backend.rs`](../../src/infrastructure/services/azure_blob_backend.rs) | + +Every entry is declared in `OXICLOUD_STORAGE_ENTRIES` (comma-separated +list of names). The active entry is stored in `admin_settings` and +switched via `oxicloud --select-storage ` on the command line +or automatically at the end of a successful `backend_migration`. +Non-active entries stay reachable through the multi-entry API (test, +audit, migrate-into). + +Adding a new backend = new struct implementing `BlobStorageBackend` ++ a new arm on `StorageBackendType` + a new branch in +`entry_backend::build_base_backend`. The wrapper stack, key rotation, +consistency check, and migration all pick it up for free (see §6). + +--- + +## 1. The `file → blob → chunk` model + +Three separate concerns, three separate storage layers: + +```text + PostgreSQL Backend + (Local/S3/Azure) + ┌─────────────────────┐ ┌──────────────────────┐ ┌──────────────┐ + │ storage.files │ │ storage.blobs │ │ │ + │ - id │────▶│ - hash (BLAKE3) │────────▶│ .blobs/xx/ │ + │ - name │ │ - size, ref_count │ │ .blob│ + │ - folder_id │ │ - content_type │ │ │ + │ - blob_hash │ │ │ │ │ + └─────────────────────┘ └──────────────────────┘ └──────────────┘ + ▲ + │ (for chunked files only) + ┌───────┴──────────────┐ + │ storage.chunk_manifests + │ - file_hash │ + │ - chunk_hashes[] │ + │ - total_size │ + └──────────────────────┘ +``` + +**File** (`storage.files`) — DB row. Has a name, a folder, a drive, an +owner, a size, a MIME type. Points at exactly one **content descriptor**: +either a whole-file blob or a chunk manifest, both keyed by BLAKE3 hash. +Files are what users see; nothing about them lives on the backend. + +**Blob** (`storage.blobs`) — DB row + **physical bytes on a backend**. +Content-addressable: the row's primary key is `hash = BLAKE3(plaintext_bytes)`. +`ref_count` is the number of live references (files or manifests) pointing +at this blob; when it reaches zero, `dedup_gc` removes both the row and +the physical bytes (after a grace window). Every backend lays blobs out +under a two-char shard directory: `.blobs//.blob` +— Local's filesystem tree, S3's object keys, Azure's blob names. See +`LocalBlobBackend::object_key` and its S3/Azure counterparts. + +**Chunk** (`storage.chunk_manifests`) — content-defined-chunking (CDC) +subdivision of a file. When an upload exceeds the whole-file threshold, +the ingest pipeline splits it into ≤1 MiB chunks and stores each as +its own blob. The manifest records the chunk sequence + total size; +downloads stream through the manifest, fetching each chunk blob in +order. **Chunk blobs are indistinguishable from whole-file blobs** at +the backend layer — they're just blobs. `storage.chunk_manifests` is +pure PG state with no backend bytes. + +**Why this matters for backend implementors:** you only ever deal with +blobs. You never see file paths, folder trees, chunks, manifests, or +users. Your API is `(hash) → put/get/exists/delete bytes`. Everything +else is orchestrated above. + +--- + +## 2. The v1 blob header (`OXCPT`) + +Every blob written since the key-rotation implementation landed +starts with a 15-byte header: + +```text +byte 0..4 "OXCPT" magic marker (5 bytes) +byte 5..6 0x00 0x01 format version (2 bytes, big-endian u16) +byte 7..14 key fingerprint (8 bytes) +``` + +Then either: +- **plaintext-v1**: `key_fp` is all zeros; the header is followed by + raw plaintext bytes. +- **encrypted-v1**: `key_fp` is `SHA-256(key_material)[..8]`; the + header is followed by a 12-byte AES-GCM nonce, then the ciphertext, + then the 16-byte GCM tag. Total overhead: **43 bytes**. + +Rendered visually via `xxd -l 15 `: + +```text +4f58 4350 54 00 01 00 00 00 00 00 00 00 00 OXCPT.......... ← plaintext-v1 +4f58 4350 54 00 01 15 f3 8f 80 2c ae 2c 50 OXCPT......,.,P ← encrypted-v1 with key_fp = 15:f3:8f:80:2c:ae:2c:50 +``` + +Fingerprints are rendered the same colon-hex form (`15:f3:…:50`) +everywhere they appear: boot log, admin panel pair chain, `xxd` +inspection, `oxicloud --fingerprint ` CLI, and the rotate / +migration audit lines. That means an admin can cross-reference by +eye — same string means same key. + +### Why the header exists + +Before the key-rotation implementation, blobs had no header. +Reading a legacy blob meant "try to decrypt with the +currently-configured key; if it works, it's encrypted; if not, +it's plaintext." This has three problems: + +1. **Ambiguity on key change.** If the operator changed the key, + every existing blob became unreadable — nothing on disk said + which key was used. +2. **No way to smart-skip.** A rotation or migration couldn't tell + whether a target blob was already in the desired state without + reading and re-hashing every byte. +3. **No forward compatibility.** Any future format change (E2E, + compression, alternate cipher) would need magic-byte detection + layered on top. + +The header solves all three. Magic bytes disambiguate legacy from +v1. Version bytes let us evolve the format. `key_fp` lets us +identify *which* key was used without trying every candidate. + +### Future: end-to-end encryption + +The current `EncryptedV1` variant is **server-side encryption at +rest** — the server holds the key. E2E encryption (client holds the +key, server sees only ciphertext) is designed to slot in as a new +version: + +```text +byte 5..6 0x00 0x02 format version = 2 (E2E) +byte 7..14 hint identifying the client key +byte 15.. client-encrypted payload, opaque to server +``` + +The read/write pipeline stays the same at the backend layer — the +server just passes bytes through. `read_dispatch` grows a match arm +for version 2 that skips server decryption entirely. This is why the +version bytes exist as a distinct field: the file format is +extensible without a magic-byte rewrite. + +### BLAKE3 rescue for legacy plaintext + +Deployments that predate the key-rotation implementation have blobs +on disk with no `OXCPT` header — the pipeline calls these `Legacy` +format. Reads try to +decrypt with each pair-list key; if all fail, a last-resort branch +computes `BLAKE3(raw_bytes)` and returns the bytes as plaintext iff +the digest matches the expected hash. Zero-false-positive by +construction (content-addressable proof). Emits +`encryption.legacy_plaintext_rescued` audit lines so operators can +spot which blobs still need re-writing. See +`encrypted_blob_backend.rs::read_dispatch` last branch. + +The rescue is transparent — downloads, thumbnails, consistency +checks, and rotation all benefit. The first time `backend_rotate` +sweeps a legacy-plaintext blob, it classifies it as `Legacy`, +rewrites it through the wrapper, and the resulting blob has a proper +v1 header. After one rotate pass, rescue never fires again. + +--- + +## 3. Key rotation + +### Pair-list config + +Encryption is configured per storage entry via a comma-separated +**pair list**: + +```bash +OXICLOUD_STORAGE__ENCRYPTION_KEY='aes_gcm:,aes_gcm:,none:' +``` + +The **head** is the leftmost entry — writes use this key. Every entry +in the list is available for reads (fallback loop). `none:` in the +list declares "raw plaintext is a legitimate on-disk shape for this +backend" — the leftmost `none:` becomes the head if placed first, +otherwise it enables the plaintext-fallback branch of `read_dispatch`. + +### Rotate job (`backend_rotate`) + +Recoverable job that iterates every blob on the current backend and +rewrites any whose header doesn't match the current head format. +Decision table via `BlobFormat::classify` compared against +`EncryptedBlobBackend::head_format`: + +| Current on-disk | Head | Action | +|---|---|---| +| `EncryptedV1 { key_fp: A }` | `EncryptedV1 { key_fp: A }` | skip | +| `EncryptedV1 { key_fp: A }` | `EncryptedV1 { key_fp: B }` | rewrite (key change) | +| `PlaintextV1` | `EncryptedV1 { key_fp: X }` | rewrite (encrypt in place) | +| `EncryptedV1` | `PlaintextV1` | rewrite (decrypt in place) | +| `Legacy` | anything v1 | rewrite (upgrade header) | + +Reports per-blob outcomes (`rewritten`, `skipped`, `failed`) and the +final head format/fp in the run's `extra_stats`. Each rewrite goes +through `put_blob_from_bytes_replace` — see §6. + +### Head-key vs fallback keys + +- **Head** — used for **writes only**. Rotating just means "declare + a new head and run `backend_rotate` to catch up existing bytes." +- **Fallback keys** — read-only. Kept in the pair list until every + blob on disk has been rewritten under the head, then safe to + remove from `.env`. + +The admin panel shows the whole pair chain per entry with the head +badged; after a successful rotation with `failed=0`, non-head keys +can be safely dropped. + +--- + +## 4. Blob consistency (`blobs_consistency`) + +Read-only recoverable job that walks `storage.blobs` and reports +divergence between the DB registry and the physical backend. + +### Shallow mode (default) + +Per row: + +- `blob_exists(hash)` on the active backend → if false, record + `blob_missing_from_backend` (severity `data_loss`) +- Compare `ref_count` against the actual reference count computed + from `SUM` over `storage.files.blob_hash` + `chunk_manifests.chunk_hashes[]` + → if mismatch, record `refcount_mismatch` (severity `inconsistent`) + +Cost: one existence probe + one aggregate SQL per row. Fast on +S3/Azure (single HEAD). + +### Deep mode (`?deep=true`) + +Adds a full read of every blob: + +- Stream the blob through `EncryptedBlobBackend::get_blob_stream` + (strips header, decrypts if needed, applies BLAKE3 rescue for + legacy plaintext) +- Recompute `BLAKE3(plaintext)` and compare against the row's `hash` +- If bytes match: no finding +- If bytes differ: record `blob_corrupted` (silent bit-rot) +- If the read pipeline errors (missing key, unreadable header, + network failure): record `blob_unreadable` + +Both `blob_corrupted` and `blob_unreadable` carry the list of files +that reference the offending hash (`affected_files`) so the operator +can decide whether to re-upload or drop. + +The `deep` flag is persisted in the run's `params` on Fresh open so +it survives a mid-run restart — resume continues in deep mode +without the operator re-specifying it. + +--- + +## 5. Backend migration (`backend_migration`) + +Recoverable job that copies every blob from the current active +backend to a target entry, then hot-swaps the active pointer on +completion. + +### Cursor + resume + +Iterates blobs in ascending hash order. Checkpoints the last-visited +hash to `jobs.recoverable_runs.cursor` after each batch. On restart, +`WHERE hash > cursor` picks up from where the previous session +stopped. `MigrationProgress` counter is seeded from +`stats.scanned_count` on resume so the maintenance banner shows +continued progress, not a fresh `0`. + +### Smart-skip via `head_check` + +Before each copy, `target.head_check(hash)` returns one of: + +- `Match` → target blob already carries the current head's + format+`key_fp`; skip. Debug-log + `backend_migration.blob_skipped_head_match`. +- `Mismatch(current_format)` → target blob exists with a different + header (legacy shape, old key, plaintext vs encrypted). Overwrite + via `put_blob_from_bytes_replace`. Info-log + `backend_migration.blob_overwritten` — this is the **legacy + skip-check residual repair** log line: exactly the blobs that + historically escaped re-encryption because the pre-rotation + migration path had an `if target exists { skip }` short-circuit + (see §5 below). +- `Absent` → fresh write. Info-log `backend_migration.blob_written`. + +The check is one 15-byte range-read via `get_blob_range_stream(hash, 0, Some(15))` +on the inner backend — cheap on Local (`pread`), cheap on S3/Azure +(single GET with `Range: bytes=0-14`). + +### Legacy skip-check residual repair (the historic bug) + +Before the key-rotation implementation, the migration path hit +`if target.blob_exists(hash) { continue }` before every copy — it +silently skipped blobs already present on the target, even if the +target's current head key was different from the key used at the +historical write time. Result: mixed-key target backends, and blobs +that suddenly failed to read when the old key was later removed +from the pair list. + +The rotation work removed the app-layer skip. Backend-agnostic +write-side fixes followed: every backend's +`put_blob_from_bytes_replace` was overridden to bypass the internal +HEAD-probe skip (`S3BlobBackend`, `AzureBlobBackend`) or use +`O_CREAT|O_TRUNC` via tempfile+rename (`LocalBlobBackend`). See §6 +for the full contract. + +### Failure gate + +`finish_completed` refuses to flip the active-backend pointer if +`failed > 0`. Emits `storage_migration.aborted` audit, clears +readonly (source stays active — writes safe there), and returns +`RunOutcome::Failed`. Operator inspects findings, then either +retries (walk short-circuits on head-format matches → cheap +re-attempt), fixes the source, or explicitly accepts the partial +via `oxicloud --select-storage `. + +--- + +## 6. Implementor contract — the `BlobStorageBackend` trait + +### Always wrapped in `EncryptedBlobBackend` + +Every entry is built via `entry_backend::build_entry_backend_typed`, +which unconditionally wraps the raw backend in `EncryptedBlobBackend` +— regardless of whether the entry has an encryption key. A `none:` +head gets an `EncryptedBlobBackend` with `head_cipher = None` that +writes plaintext-v1 blobs. This means: + +- Every read goes through `read_dispatch` → magic-byte inspection → + v1 or legacy branch → decrypt (if needed) → BLAKE3 rescue (if + legacy plaintext). +- Every write goes through the wrapper's write path → prepend the + 15-byte header → encrypt with head cipher (or leave plaintext) → + hand to inner backend. +- The **inner backend never sees plaintext application content** — + only the header-wrapped or encrypted body. + +Your job as a backend implementor: implement `BlobStorageBackend` +for opaque byte payloads. Never inspect or modify the bytes. + +### Required overrides + +The trait provides defaults but they lie about correctness for the +rotate/migrate use case. **Every production backend MUST override +`put_blob_from_bytes_replace`.** + +| Method | Semantics | Default | Must override? | +|---|---|---|---| +| `put_blob` | Content-addressed upload. May skip if hash exists (dedup, idempotency). | — | yes | +| `put_blob_from_bytes` | Same, in-memory bytes. May skip if exists. | — | yes | +| `put_blob_from_bytes_unsynced` | Unconditional PUT. Durability not required on return; caller batches `sync_blobs`. | delegates to `put_blob_from_bytes` (WRONG for skip-backends) | recommended (dedup fast-path) | +| **`put_blob_from_bytes_replace`** | **Unconditional overwrite. Durable on return.** Used by rotate + migration. | delegates to `put_blob_from_bytes` (WRONG for every current backend) | **yes** | +| `get_blob_stream` | Full-blob stream. | — | yes | +| `get_blob_range_stream` | Range stream — used by the 15-byte `head_check` probe. | — | yes | +| `blob_exists` | Cheap presence check. | — | yes | +| `delete_blob` | Physical deletion. | — | yes | +| `sync_blobs` | Fsync barrier for `_unsynced` writes. | no-op | Local only | +| `initialize` | Called at boot. Verify creds, create shard dirs, reap tempfiles. | no-op | yes | + +The `put_blob*` skip-if-exists semantics is correct for **uploads** +— dedup hits should short-circuit. It's wrong for **rewrites** — +rotate needs to overwrite the header, migration needs to overwrite +with new-key ciphertext. `put_blob_from_bytes_replace` is the +escape hatch. On S3/Azure it delegates to `put_blob_from_bytes_unsynced` +(unconditional PUT, durable on return). On Local it does +write-to-tempfile + `rename(2)` + fsync — atomic overwrite on POSIX. + +### Reference implementations + +- [`LocalBlobBackend`](../../src/infrastructure/services/local_blob_backend.rs) + — filesystem tree under a configurable root. Shard directory + `.blobs//`. Tempfiles named `.replace...tmp`, + reaped at boot by `initialize`. +- [`S3BlobBackend`](../../src/infrastructure/services/s3_blob_backend.rs) + — AWS SDK v2. Object key `/.blob` under a configurable + bucket. `put_blob_from_bytes_replace` delegates to + `put_blob_from_bytes_unsynced` (skips the HEAD probe). +- [`AzureBlobBackend`](../../src/infrastructure/services/azure_blob_backend.rs) + — Azure SDK. Same shape as S3. + +Read them side-by-side before implementing a new backend — the three +follow the same skeleton so the diff is where your backend's +semantics genuinely differ. + +--- + +## Related docs + +- [File and blob lifecycle →](./file-and-blob-lifecycle.md) — the + hook system that observes file/blob CRUD events. +- [Background jobs →](./jobs.md) — how to plug in a new job tenant. +- [Storage quotas →](./storage-quotas.md) — usage accounting layer + (independent of the backend). +- [Storage multi-entry plan →](../plan/storage-multi-entry.md) — + design history: why multiple entries, why hot-swap migration. +- [Storage key rotation plan →](../plan/storage-key-rotation.md) — + design history and slice-by-slice implementation notes. diff --git a/docs/architecture/index.md b/docs/architecture/index.md index 29df37d9..62f12d10 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -74,4 +74,5 @@ src/ - [Caching Architecture →](/architecture/caching) - [Resource Listing API →](/architecture/resource-listing) - [Storage Quotas →](/architecture/storage-quotas) +- [Backend Storage →](/architecture/backend-storage) - [Background Jobs →](/architecture/jobs) diff --git a/docs/architecture/jobs.md b/docs/architecture/jobs.md index 5195237e..f045d1c6 100644 --- a/docs/architecture/jobs.md +++ b/docs/architecture/jobs.md @@ -41,7 +41,7 @@ operator benefit. | Job name | Cadence | Force semantic | Service | |---|---|---|---| | `trash_cleanup` | 24 h (hardcoded in DI, no env var yet) | ignored | [`trash_cleanup_service.rs`](../../src/infrastructure/services/trash_cleanup_service.rs) | -| `storage_reconcile`| `OXICLOUD_STORAGE_USAGE_RECONCILE_SECS` (default 600s, min 30s) | ignored | [`storage_usage_service.rs`](../../src/application/services/storage_usage_service.rs) | +| `usage_reconcile` | `OXICLOUD_STORAGE_USAGE_RECONCILE_SECS` (default 600s, min 30s) | ignored | [`storage_usage_service.rs`](../../src/application/services/storage_usage_service.rs) | | `dedup_gc` | on-demand only (trash cleanup runs it inline as its tail step) | `force=true` → `garbage_collect_force()` (skip orphan grace) | [`dedup_service.rs`](../../src/infrastructure/services/dedup_service.rs) | | `grant_cleanup` | `OXICLOUD_GRANT_CLEANUP_INTERVAL_HOURS` (default 24h) — feature-gated by `OXICLOUD_GRANT_CLEANUP_ENABLED` | `force=true` → `purge(Some(0))` (grace_days=0) | [`grant_cleanup_service.rs`](../../src/infrastructure/services/grant_cleanup_service.rs) | diff --git a/docs/config/admin-settings.md b/docs/config/admin-settings.md index e874f81e..0dc1f082 100644 --- a/docs/config/admin-settings.md +++ b/docs/config/admin-settings.md @@ -79,7 +79,7 @@ The admin storage tab operates on the **named storage entries** declared in `.en | `POST` | `/api/admin/storage/migration/resume` | Resume a paused run (target read from `params.target_name`, no body needed) | | `GET` | `/api/admin/storage/migration` | Poll the current run's progress | -Runs are recoverable — status, cursor, and per-blob failure findings all live in `jobs.recoverable_runs` / `jobs.run_findings`. The same run history is browsable via `GET /api/admin/jobs/storage_migration/runs`. +Runs are recoverable — status, cursor, and per-blob failure findings all live in `jobs.recoverable_runs` / `jobs.run_findings`. The same run history is browsable via `GET /api/admin/jobs/backend_migration/runs`. ### Cutover flow (moving the active pointer) diff --git a/docs/config/env.md b/docs/config/env.md index c39651e7..05febfdb 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -103,8 +103,7 @@ Each declared name `` then reads its own set of per-entry variables: | `OXICLOUD_STORAGE__AZURE_CONTAINER` | — | Azure-only: blob container name (required when backend=azure) | | `OXICLOUD_STORAGE__AZURE_SAS_TOKEN` | — | Azure-only: SAS token (alternative to account key) | | `OXICLOUD_STORAGE__AZURE_ENDPOINT_URL` | — | Azure-only: custom endpoint (Azurite, private deployments) | -| `OXICLOUD_STORAGE__ENCRYPTION_KEY` | — | Base64-encoded 32-byte AES-256 key. **Presence implies encryption is enabled** on this entry — no separate enable flag. Bad base64 / wrong length aborts boot. | -| `OXICLOUD_STORAGE__ENCRYPTION_CIPHER` | `aes-256-gcm` when `_ENCRYPTION_KEY` is set | Cipher choice for this entry. Only `aes-256-gcm` is accepted today (future-proofing knob — the enum is ready for a second cipher, the implementation still hardcodes AES-256-GCM). Setting the cipher without a key aborts boot. | +| `OXICLOUD_STORAGE__ENCRYPTION_KEY` | — | Comma-separated list of `:` pairs (or bare ``, which defaults to `aes-256-gcm`). **Presence implies encryption is enabled** on this entry — no separate enable flag. The LAST pair wins on writes; every pair is a candidate for reads. Supported ciphers: `aes-256-gcm` and `none` (empty-key sentinel used at pair-list head/tail for encrypt/decrypt-in-place rotations). Bad base64, wrong length, duplicate keys, or multiple `none` pairs abort boot. See [Storage key rotation](../plan/storage-key-rotation.md) for rotation recipes. | **Fail-fast rules** (boot aborts with actionable message): @@ -125,7 +124,10 @@ OXICLOUD_STORAGE_s3_prod_S3_BUCKET=my-oxicloud-bucket OXICLOUD_STORAGE_s3_prod_S3_REGION=us-east-1 OXICLOUD_STORAGE_s3_prod_S3_ACCESS_KEY=… OXICLOUD_STORAGE_s3_prod_S3_SECRET_KEY=… -OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=… # openssl rand -base64 32 +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:… # openssl rand -base64 32 + +# Rotation window (two pairs, last wins on writes): +# OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: ``` ## Storage Backend (DEPRECATED — legacy single-backend) diff --git a/docs/guide/backend-storage.md b/docs/guide/backend-storage.md index 6735711a..19d04bdd 100644 --- a/docs/guide/backend-storage.md +++ b/docs/guide/backend-storage.md @@ -40,15 +40,78 @@ A few things to know: Any backend can be encrypted at rest by adding an encryption key to it: ``` -OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=… # base64 of 32 random bytes +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:… # base64 of 32 random bytes ``` A key can be generated from **Settings → Storage → Generate key** in the admin panel. Set the same key on a new backend during migration and OxiCloud re-encrypts the data as it copies. +The bare shorthand `OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=` (no `aes-256-gcm:` prefix) also works and is treated as AES-256-GCM. Use the explicit form once you have more than one key in the list — see [Rotating an encryption key](#rotating-an-encryption-key) below. + ::: warning If you lose the encryption key, the data encrypted with it is unrecoverable. Store the key somewhere as safe as you'd store a database backup. ::: +### Rotating an encryption key + +When it's time to rotate a key — after a suspected leak, on a periodic policy, or after a staff turnover — you don't need to provision a second bucket. Add the new key **alongside** the old one; the server writes with the new key while reads keep working through the old one; then a background job re-encrypts existing data under the new key; then you drop the old key. + +Step by step: + +1. Generate a new key: + + ``` + openssl rand -base64 32 + ``` + +2. Append it to the entry's key list. **Order matters — the NEW key goes LAST.** + + ``` + OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: + ``` + +3. Restart the server. New uploads are now encrypted with the new key; existing files still open normally because the old key is still in the list. + +4. On the **Storage** tab, click **Rotate encryption key** on the entry. This dispatches a background job that re-encrypts every existing file under the new key. All operations keep working during rotation — uploads, browsing, downloads, sharing. + +5. Wait for the job to complete. Progress shows in a top banner and on the **Jobs** tab. + +6. Once the entry card says "*Rotation complete — safe to remove the old key*", remove the OLD key from the list: + + ``` + OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm: + ``` + +7. Restart the server. Rotation is done. + +::: warning +Do NOT remove the old key before the rotation job reports "safe to remove". Any file still encrypted under the old key would become unreadable. +::: + +### Encrypting a backend that started unencrypted + +Same shape as key rotation, using the `none:` sentinel to represent "the current writes are plaintext": + +1. Generate a new key. +2. Add it AFTER `none:`: + + ``` + OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY=none:,aes-256-gcm: + ``` + +3. Restart. New uploads are encrypted; existing plaintext files stay readable. +4. Trigger **Rotate encryption key** on the entry. +5. Once the entry card says the rotation is done, remove `none:` from the list; restart. + +### Decrypting an encrypted backend + +The symmetric flow — add `none:` AFTER the current key, restart, rotate, drop the old key: + +``` +OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY=aes-256-gcm:,none: +``` + +After the rotation job completes and the entry card says done, remove the AES pair (or the whole variable) and restart. All files on that entry are now plaintext. + ## Checking a backend Once a backend is declared, it appears on the admin **Storage** tab as a card: diff --git a/docs/plan/consistency-check.md b/docs/plan/consistency-check.md index 7b864917..ad9fe1f7 100644 --- a/docs/plan/consistency-check.md +++ b/docs/plan/consistency-check.md @@ -565,7 +565,7 @@ CREATE SCHEMA IF NOT EXISTS admin; CREATE TABLE jobs.recoverable_runs ( id UUID PRIMARY KEY, - job_name TEXT NOT NULL, -- 'consistency_blobs', 'storage_migration', 'reextract_audio', ... + job_name TEXT NOT NULL, -- 'consistency_blobs', 'backend_migration', 'reextract_audio', ... status TEXT NOT NULL, -- Running / Paused / Completed / Failed / CancelRequested started_at TIMESTAMPTZ NOT NULL, last_progress_at TIMESTAMPTZ NOT NULL, diff --git a/docs/plan/job-registry.md b/docs/plan/job-registry.md index 0a82658e..ebc30e6e 100644 --- a/docs/plan/job-registry.md +++ b/docs/plan/job-registry.md @@ -716,7 +716,7 @@ rationale + the merges/separations that fall out of the rule. | `files_consistency` | `storage.files` | file UUID | `parent_folder_trashed` (live file under trashed folder), `missing_blob` (severity `data_loss` — `blob_hash` present in neither `storage.blobs` nor `storage.chunk_manifests`), `chunk_missing` (severity `data_loss` — manifest exists but points at chunks absent from `storage.blobs`; typical dedup GC race), `blob_size_mismatch` (denormalised `files.size` diverges from the authoritative size — manifest first, blob fallback) | Shipped Slice 6, CDC-aware Slice 10. Handles both storage paths: `storage.chunk_manifests` (post-Apr-2026 FastCDC ingest, dominant path) and `storage.blobs` (pre-CDC whole-file blob, legacy fallback). Physical backend-existence checks (chunk bytes actually on disk) belong in `storage_consistency`. Room to grow: `drive_id_parent_mismatch`, mime-type reconciliation. | | `storage_consistency` | Storage backend (fs / S3) | object key / path | Each blob has a `storage.blobs` row (orphan detection) | `?deep=true` adds re-BLAKE3 + mime sniff. Orphan-side of the old bidirectional blob check + former `blob_integrity` + former `thumbnail_consistency`. | | `grants_consistency` (future) | `storage.role_grants` | grant UUID | subject/resource/granted_by exist | | -| `storage_migration` | `storage.blobs` (source) → target backend | blob hash | Copy bytes; failures → `stats.failed_blobs` (and eventually `jobs.run_findings`) | Retires `Arc>` in `migration_job.rs`. | +| `backend_migration` | `storage.blobs` (source) → target backend | blob hash | Copy bytes; failures → `stats.failed_blobs` (and eventually `jobs.run_findings`) | Retires `Arc>` in `migration_job.rs`. | | `reextract_audio` | `storage.files` where audio | file UUID | Re-run audio-tag parser, upsert `audio_metadata` | Retires synchronous admin-request execution. | | `reextract_image` | `storage.files` where image/video | file UUID | Re-run EXIF/container date parser, upsert capture date | Same shape as reextract_audio. | | `consistency_batch` (wrapper) | Iterates registered `*_consistency` jobs | — (JobHandler, not RecoverableJobHandler) | Sequentially triggers each sub-job; `?deep=true` propagates | Shipped Slice 5. One-click "run all" without per-job clicks; exclusivity via `job_name` prevents concurrent batches from stepping on each other. Batch itself always returns `Ok` — child failures land in `outcome.extra.per_check[].outcome`. | @@ -729,7 +729,7 @@ SELECT + one UPDATE. Kept as its own admin endpoint; do NOT fold into ### Verification (Part 2) 1. **Compile + schema-migration idempotence.** -2. **Fresh run:** `POST /api/admin/jobs/storage_migration/trigger` → new row with +2. **Fresh run:** `POST /api/admin/jobs/backend_migration/trigger` → new row with `status='Running'`, `cursor=NULL`. 3. **Concurrent trigger:** second `POST` while the first is running returns the SAME `run_id` (idempotent, DB unique index enforces). @@ -787,7 +787,7 @@ reference for any external tool that still expects the old paths: | Legacy (retired) | Replacement | |---|---| -| `POST /admin/internal/trigger-sweep` | `POST /admin/jobs/storage_reconcile/trigger` | +| `POST /admin/internal/trigger-sweep` | `POST /admin/jobs/usage_reconcile/trigger` | | `POST /admin/internal/trigger-gc?force=X` | `POST /admin/jobs/dedup_gc/trigger?force=X` | | `POST /admin/internal/trigger-grant-cleanup?force=X` | `POST /admin/jobs/grant_cleanup/trigger?force=X` | @@ -816,7 +816,7 @@ complete. Rough shape: │ Name Cadence Last run Status Actions │ │ ───────────────────────────────────────────────────────────────────│ │ trash_cleanup every 24 h 3h ago ok [Run] │ -│ storage_reconcile every 10 m 4m ago ok [Run] │ +│ usage_reconcile every 10 m 4m ago ok [Run] │ │ dedup_gc on-demand 1d ago ok [Run] │ │ grant_cleanup every 24 h never — [Run] │ │ drives_consistency on-demand never — [Run] │ diff --git a/docs/plan/storage-key-rotation.md b/docs/plan/storage-key-rotation.md new file mode 100644 index 00000000..7b908f3b --- /dev/null +++ b/docs/plan/storage-key-rotation.md @@ -0,0 +1,609 @@ +# Plan — Storage encryption-key rotation + +## Context + +Today an encrypted storage entry declares a single key: + +```env +OXICLOUD_STORAGE_s3_prod_BACKEND=s3 +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY= +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_CIPHER=aes-256-gcm # optional, defaults to aes-256-gcm +``` + +Blobs land at `.blob` as raw AES-GCM output (`nonce | ciphertext | tag`) — +no discriminating prefix. That key can never change. Any real deployment needs +to rotate keys — after a suspected leak, on a periodic policy, or after a staff +turnover. The current answer is "create a second entry and run a migration to +it," which: + +* forces the admin to provision a second bucket / directory, +* churns object storage costs, +* and does not scale to routine rotations. + +The proper fix is **in-place key rotation** inside the same entry. The storage +config holds a *list* of keys, the server writes with the newest one, reads +with any of them, and a background job re-encrypts every blob under the newest +key. When the job completes, the admin removes the old key from the list. + +At the same time we solve two related on-disk problems that today's format +leaves open: + +1. **No self-description.** A `.blob` today is ambiguous: could be raw + plaintext (unencrypted deployment) or raw AES-GCM output (encrypted + deployment). Reads guess based on the entry config; a misconfigured key or + a wrong-cipher default silently returns garbage. +2. **No path from encrypted to plaintext.** Once encryption is on for an + entry, the only way off is via a second entry and full migration — same + ergonomics problem as key rotation. + +Both fall out of introducing a self-describing on-blob header (`v1`). The +object-key suffix stays `.blob` for both eras — the magic bytes at the +top of every v1 blob are the discriminator, so legacy files stay readable +byte-identically and new files coexist alongside them at the same suffix. +Migration is lazy. + +Related memory (this plan supersedes it): + +> **[Encryption key rotation in place — deferred, unsafe today]** — the previous +> "namespace object keys by encryption generation" direction (`.k2.blob`) +> is DROPPED in favour of the pair-list + v1-header approach below. That +> approach fought content-addressability by generation-namespacing per key; +> this one keeps a stable content-addressable object key and encodes generation +> in a compact on-blob header. + +## Design + +### The pair-list config + +Replace the singular `_ENCRYPTION_KEY` + `_ENCRYPTION_CIPHER` with a single +comma-separated list of pairs on `_ENCRYPTION_KEY` alone: + +```env +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: +``` + +Rules: + +* **Format** — `[:]` per pair, comma-separated. Whitespace + around commas / colons tolerated. +* **Cipher optional.** Only one real cipher (`aes-256-gcm`) exists today, so + `` on its own is legal and behaves as `aes-256-gcm:`. When a + second cipher lands, the colon-prefixed form is the disambiguator. +* **The `none` cipher.** Legal, has no key material (`none:` — trailing colon + with an empty key). Enables migrating BOTH ways: encrypt a previously-plaintext + deployment (`none:,aes-256-gcm:`), or decrypt an encrypted deployment + (`aes-256-gcm:,none:`). Correctness relies on the v1 header (see + *Encrypted-blob format*) discriminating encrypted vs plaintext, NOT on + fallback ordering. A `none` pair by itself in the list is equivalent to + omitting `_ENCRYPTION_KEY` entirely (kept for symmetry). +* **Order matters. Last pair wins on writes.** For v1-header reads, the + `key_fp` in the header selects the exact pair; the list order is irrelevant + at read time. For legacy reads (see *Coexistence*), the head pair is used + because pre-v1 deployments had only one key by construction. +* **At least one pair.** Empty list → boot aborts. +* **Uniqueness.** Same key appearing twice = boot aborts (config drift smell). + `none` may appear at most once. +* **Fingerprint.** For each real-cipher pair, log a truncated SHA-256 of the + key material at boot (12 hex chars) so a swap-order accident is loud in the + audit stream. `none` logs as `none:—`. + +### Retire `_ENCRYPTION_CIPHER` + +`_ENCRYPTION_CIPHER` was added recently and has not shipped in a release. Drop +it entirely in the same slice — the pair-list makes it redundant, and the on-blob +v1 header carries the cipher choice explicitly per blob anyway. No back-compat +shim needed. + +### Encrypted-blob format (v1 header) + +Every v1 blob starts with a fixed self-describing header. Layout: + +``` +"OXCPT" 5 bytes — magic marker (0x4F 0x58 0x43 0x50 0x54) + 2 bytes — big-endian u16; v1 = 0x0001 + 8 bytes — sha256(key material)[..8]; identifies which pair + decrypts. For a `none` (plaintext) v1 blob this is + eight zero bytes and the fields below are absent. + 12 bytes — random per blob (AES-GCM standard nonce length) + N bytes — encrypted payload; same length as the plaintext + 16 bytes — AEAD authentication tag + ──── +fixed overhead: 43 bytes for encrypted v1; 15 bytes for plaintext v1 +``` + +* **The magic `OXCPT`** unambiguously distinguishes a v1 blob from legacy raw + bytes. It's the sole discriminator — no filename convention, no DB column, + the first five bytes tell the reader everything. +* **``** is the only field that can change format without touching + every blob. v1 is what this plan ships; v2 slots are reserved + (see *Forward compatibility*). +* **``** eliminates guess-work on read. The pair-list lookup becomes + O(1) — no fallback tag-check loop. A key-fp with no matching pair means + `NoKeyForBlob` (500 with a clear message), NEVER silent garbage. +* **``** and **``** are the AES-GCM primitive's own outputs. + Their length is fixed by v1 = AES-256-GCM. If a future v2 uses a different + AEAD with the same shape (say ChaCha20-Poly1305, tag = 16 B, nonce = 12 B) + v2 can keep the layout; a truly-different AEAD triggers a new version. +* **Plaintext v1 blobs** (produced when the entry's head pair is `none`) use + the same magic + version + all-zero key_fp, then the raw plaintext bytes + directly (no nonce, no ciphertext framing, no tag). The header alone is + enough to say "this is plaintext v1", cleanly distinguished from encrypted + v1 and from legacy raw-plaintext. + +**Why no CRC.** The AEAD tag covers ciphertext + nonce integrity. Content +addressability (`` = BLAKE3(plaintext)) covers whole-blob integrity for +either format. A CRC over the header adds nothing — a corrupt version byte +fails on decode; a corrupt key_fp fails to find a pair. Both fail loud. + +### Coexistence with legacy blobs + +Legacy and v1 blobs share the `.blob` object-key namespace. The magic +bytes at position 0 discriminate them safely: + +| First 5 bytes | Interpretation | Read path | +|---|---|---| +| `"OXCPT"` | v1 blob | verify magic, dispatch on version + key_fp | +| anything else | legacy | apply entry config: raw plaintext if no encryption pair, single-key AES-GCM decode with the head pair if encryption is set (by construction pre-v1 used one key, and any pair-list produced by upgrading from a pre-v1 config keeps that key at the head) | + +Collision probability of legacy bytes accidentally matching `"OXCPT"` at +position 0 is 2⁻⁴⁰ per blob. If a match does happen, the read continues into +the v1 path, fails at version/key_fp check with `UnsupportedBlobVersion` or +`NoKeyForBlob`, and returns a hard error. **Never silent corruption.** For +context: on a 10 M-blob deployment, expected number of collision hard-errors +across the lifetime of legacy files is ~10⁻⁵ blobs. Effectively never. + +Properties this buys us: + +* **No forced migration.** Legacy `.blob` files stay readable + indefinitely. Existing deployments upgrade to v1 code with zero + format-conversion work. +* **Lazy conversion on hot paths.** Any COW overwrite (WebDAV MOVE, PUT-over, + content-hash re-upload) naturally lands as v1 at the same object key. +* **Explicit conversion via `backend_rotate`.** The rotate job walks + `storage.blobs`, reads each blob via the magic-byte dispatch, and if the + blob is not already v1 with the head-pair key, PUTs it back as v1 with + the head pair — in place, same object key. +* **No schema changes.** Nothing new in `storage.blobs`; nothing for + `blobs_consistency`, `storage_migrate`, or any other sibling job to learn. + +### Legacy-pair guardrail + +Once any legacy blob remains on disk, the pre-v1 single key that produced +those blobs MUST remain in the pair-list at the head position (or in the list +somewhere the legacy read path can find it). Removing it early breaks every +legacy read. + +Guardrail: + +* Boot logs a `warn` line if any pair-list has more than one entry AND the + head pair's fingerprint differs from the second entry's — signals "you + added new keys but haven't rotated legacy blobs yet". +* A **legacy-blob counter** is surfaced in the admin panel per storage entry. + The counter is maintained by `blobs_consistency`: during its normal walk it + branches on the magic-byte check and records the legacy count as a run + statistic on `jobs.recoverable_runs` (existing surface, no schema hit). The + admin panel reads the most recent count and displays it. Refresh cadence is + whatever consistency scan cadence the deployment has (weekly by default; on + demand from the admin panel). +* The *"Rotation complete — safe to remove the old key"* hint appears in the + entry card only when the last `backend_rotate` run completed with zero + findings AND the most recent consistency scan reported zero legacy blobs. +* Nothing enforces removal at code level — the admin is trusted, given a + clear signal, and warned. + +### Read path + +Per blob read: + +1. Fetch `.blob` from the backend. +2. Check first 5 bytes. +3. If `"OXCPT"` → v1 read path: + * Read ``. Not `0x0001` → `UnsupportedBlobVersion`. + * Read ``. If zero → plaintext-v1; return raw bytes (post-header + payload). Else find the pair whose `sha256(key)[..8]` matches; not + found → `NoKeyForBlob`; found → AES-GCM decrypt with `` / + `` / `` and return plaintext. +4. Otherwise → legacy read path: + * If entry has any real-cipher pair, attempt AES-GCM decrypt with the + head pair's key; return plaintext on success, `DecryptFailed` on tag + failure. + * If entry has only `none` (or no pair at all), return raw bytes as + plaintext. + +Never falls through silently. Every failure is a distinct typed error the +handler can map to a 500 with an actionable message. + +### Write path + +1. Always writes to `.blob`. +2. Head pair is `none` → write v1 header (magic + version + zero key_fp) + + raw plaintext. +3. Head pair is a real cipher → compute AEAD; write v1 header (magic + + version + key_fp) + nonce + ciphertext + auth_tag. + +v1 code never produces a legacy-format blob. Any leftover legacy blobs on +disk pre-date the v1-code deployment. + +### The rotation job + +New `RecoverableJobHandler` tenant, `backend_rotate`. Mirrors +`backend_migration`'s shape: + +* **Iterates `storage.blobs`** in hash-lex order. Cursor is the last-processed + hash (64 hex chars). Same cursor encoding as `backend_migration` and + `blobs_consistency`. +* **Per blob:** + 1. Fetch `.blob` and dispatch via the standard read path. + 2. Decide if a rewrite is needed based on what actually decoded: + * Legacy blob (no v1 magic) → always rewrite (upgrade to v1 with head + pair). + * v1 encrypted, decrypted under a pair-index other than head → rewrite + (key rotation). + * v1 encrypted, already under head — skip. + * v1 plaintext, head pair is `none` — skip. + * v1 plaintext, head pair is a real cipher — rewrite + (encrypt-in-place upgrade). + * v1 encrypted, head pair is `none` — rewrite + (decrypt-in-place downgrade). + 3. Write via the standard v1 write path — same object key, atomic + replace. + 4. Checkpoint. On per-blob failure, record a `rotation_failed` finding + with severity `data_loss` (bytes may not have crossed), continue. +* **Concurrency-safe by construction.** + * A concurrent upload during rotation writes v1 with the head key. The + rotate walk will short-circuit on that hash if it reaches it later. + * A COW overwrite is the same story. + * The v1 write is atomic at object-storage level (S3 replace, Local + rename-into-place). A concurrent reader sees either state. + * No readonly mode. This is a critical improvement over + `backend_migration`: rotation is per-blob idempotent, so we don't need + to freeze writes. +* **Restart-survivable** — same boot-time sweep as every other recoverable + handler. +* **`?deep=true` unused** — rotation has no "slow variant" mode. Parameter + accepted for uniformity, ignored. + +### Admin UX + +In the admin storage-panel entry card, add a **Rotate encryption key** action. +Preconditions: + +* Entry has encryption enabled with ≥ 2 pairs OR entry has legacy blobs + outstanding OR the pair-list has otherwise changed since the last rotation. +* Otherwise the button is disabled with the tooltip *"Add a second pair in + `OXICLOUD_STORAGE__ENCRYPTION_KEY` first, or wait for legacy blobs to + accumulate — nothing to rotate right now."* + +Clicking the button dispatches the `backend_rotate` job for that entry. The +job's progress rides on the same `X-Server-Status` header infrastructure the +maintenance banner uses — but this time WITHOUT engaging read-only mode. +Banner variant reads *"Rotating encryption key on `` — X% (Y / Z +blobs). All operations continue normally."* and disappears on completion. + +The entry card shows a **legacy-blob counter** sourced from the most recent +`blobs_consistency` run: *"N legacy-format blobs remain (upgrade included in +next rotation)"*. Refresh-on-demand button next to it triggers a targeted +`blobs_consistency` scan (already available via the admin surface). The +*"Rotation complete — safe to remove the old key"* hint appears only when +N = 0 and the last `backend_rotate` run completed with zero findings. + +### Removing the old pair + +Not automated. The admin edits `.env`, restarts. Rationale: + +* We do not want the running server to silently mutate its own `.env`. +* A retained old pair is a benign cost (nothing hits it on read since + `key_fp` lookup is O(1)). +* Explicit human step matches the "add a new pair → restart" symmetry. + +## Deployment flow + +### First-time upgrade to v1 + +Zero admin work required. On upgrade: + +* v1 code deploys. +* New blobs are written in v1 format at the existing `.blob` object + key. +* Existing legacy blobs stay readable via the magic-byte dispatch — the + legacy read path is preserved verbatim. +* Admin can optionally trigger a `backend_rotate` run to consolidate every + legacy blob into v1 format. Not required — legacy blobs migrate + opportunistically via COW overwrites and stay readable indefinitely + otherwise. + +### Rotating an encryption key + +The user-facing recipe (goes verbatim into `docs/guide/backend-storage.md`): + +``` +1. Generate a new key: + openssl rand -base64 32 + +2. Append it to the entry's key list. Order matters — the NEW key goes LAST: + OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: + +3. Restart OxiCloud. New uploads are now encrypted with the new key; existing + blobs still decrypt with the old one. + +4. In the admin panel, click "Rotate encryption key" on the entry. This + dispatches a background job that re-encrypts every existing blob under + the new key AND upgrades any remaining legacy-format blobs to v1. All + operations keep working during rotation. + +5. Wait for the job to complete (progress shows in the top banner and in + the Jobs admin page). + +6. Remove the OLD key from the list: + OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm: + +7. Restart OxiCloud. Rotation is complete. +``` + +### Encrypting a previously-plaintext deployment + +``` +1. Generate a new key (as above). +2. Add it AFTER `none`: + OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY=none:,aes-256-gcm: +3. Restart. New uploads are encrypted; existing plaintext blobs stay readable. +4. Run `backend_rotate` to encrypt existing blobs in place. +5. Remove `none:` from the list; restart. +``` + +### Decrypting an encrypted deployment + +``` +1. Add `none:` AFTER the current encryption key: + OXICLOUD_STORAGE_local_main_ENCRYPTION_KEY=aes-256-gcm:,none: +2. Restart. New uploads are plaintext; existing encrypted blobs stay readable. +3. Run `backend_rotate` to decrypt existing blobs in place. +4. Remove the key pair, keep `none:` only (or drop `_ENCRYPTION_KEY` entirely); + restart. +``` + +## Testing strategy + +**Most of this ships as Rust tests, not Hurl.** The rotation surface is +byte-level, cryptographic, and stateful across restart boundaries — none of +which Hurl can meaningfully assert. Push almost everything down into +`#[cfg(test)]` blocks alongside the code, following the codebase's existing +inline-tests convention (see AGENTS.md — "tests are primarily `#[cfg(test)]` +modules within source files"). + +**What lives in Rust (unit + integration):** + +* **Pair-list parser.** Unit tests in `config.rs` on `_ENCRYPTION_KEY` parsing: + 1-pair, 2-pair, cipher-optional shape, `none:` alone, `none:,aes:K`, + `aes:K,none:`, whitespace tolerance, duplicate rejection, multiple-`none` + rejection, empty-list rejection, retired-`_ENCRYPTION_CIPHER` rejection, + base64 length validation. Table-driven; fast. +* **v1 header round-trip.** Unit tests in + `encrypted_blob_backend.rs::tests`: encrypt payload → verify header bytes + (magic, version, key_fp) → decrypt → assert plaintext identity. One test per + format flavour (encrypted, plaintext-with-`none`-head). Verify `` = + BLAKE3(plaintext) with no offset for plaintext-v1 (the "recovery tool" + invariant). +* **Legacy read-path preservation.** Fixture: a hand-crafted raw AES-GCM + ciphertext (`nonce | ct | tag`) produced by the pre-v1 code path. Assert + that magic-byte dispatch falls through to the legacy branch and decrypts + correctly. Fixture stays in-tree as `tests/fixtures/legacy_blob.bin` so + regressions on the legacy path can never sneak through. +* **`OXCPT` collision on random data.** Property test: generate random N-byte + payloads, feed through legacy read path. Assert the ~2⁻⁴⁰ magic-collision + case fails with `UnsupportedBlobVersion` or `NoKeyForBlob` — hard error, + never silent misread. Keeps the "collisions can't silently corrupt" claim + in the plan honest. +* **Rotation decision tree.** Unit tests on the per-blob `decide()` helper of + `backend_rotate_service.rs`. All six cases from *The rotation job* section + as separate tests with clear names (`legacy_upgrades_to_v1`, + `v1_encrypted_under_head_skips`, + `v1_encrypted_under_older_pair_rewrites`, + `v1_plaintext_with_none_head_skips`, + `v1_plaintext_encrypts_when_head_is_cipher`, + `v1_encrypted_decrypts_when_head_is_none`). +* **Recoverable-job round-trip.** Integration test in + `backend_rotate_service::tests` using the existing recoverable-run harness: + seed N blobs (mix of legacy + v1-under-old-key), trigger rotation, assert + every blob ends v1-with-head, `format_generation` (if we add it later) or + the consistency-scan count reports zero legacy remaining, findings=0. +* **Crash recovery.** Same harness: interrupt mid-run at cursor position K, + restart, assert resume from K and eventual completion with correct final + state. Same discipline `backend_migration` already uses. +* **Concurrency safety.** Test that a `put_blob` call during a rotation + targeting the same hash produces exactly one v1 blob at end-state (either + the rotate's or the concurrent write's — both are head-format so the final + state is indistinguishable). Verifies the "per-blob idempotent" claim. +* **Config-restart semantics.** Table-driven test on the pair-list state + machine: start with pair-list [K1], add K2 → [K1,K2], run rotation, + drop K1 → [K2], assert every blob still decrypts. Exercises the "safe to + remove old key" transition end-to-end without going through the admin UI. + +**What Hurl covers (thin — the API surface, not the mechanics):** + +* Trigger endpoint auth: `POST /api/admin/storage/entries/{name}/rotate` is + admin-only (403 for non-admins, 401 for unauthenticated). +* Trigger endpoint preconditions: 400 or similar when pairs < 2 AND no + legacy blobs exist AND pair-list hasn't changed. +* Trigger endpoint concurrency: second POST while a run is Active returns + 409. +* Progress header: `X-Server-Status` includes a `rotation` payload during a + running job and drops it on completion. + +Hurl does NOT try to: +* Inspect on-disk bytes. +* Restart the server with a new pair-list. +* Seed a legacy blob directly. +* Verify decryption after old-key removal. + +Those all belong in Rust tests where we can hold the pool, the backend, and +the config in the same test's memory. + +**Test data.** Legacy-blob fixtures are byte-frozen (`tests/fixtures/*.bin`) +and committed. Don't regenerate them from live code — the whole point is that +they were produced by pre-v1 code and won't ever again be. Regeneration +scripts (kept outside `tests/`) are OK for one-time refreshes if the format +changes. + +## Slices + +Same discipline as `storage-multi-entry.md`. Each slice is a mergeable +increment; no slice depends on a later one. + +### Slice K1 — Pair-list config parsing + +**Scope.** Config layer only. No behaviour change on the write / read path +yet — the parser produces `Vec` and the existing code keeps calling +`pairs.last()`. + +* `NamedStorageEntry.encryption` becomes `Option>` where + `KeyPair = { cipher: CipherKind, key_material: Option<[u8; 32]> }`. + `CipherKind` = `{ AesGcm256, None }`; `None` carries no key. +* Parser: split on `,`, per-pair split on `:` (1 or 2 parts), decode base64 + when a real cipher, reject empty list, reject duplicates, reject multiple + `none`. +* Retire `OXICLOUD_STORAGE__ENCRYPTION_CIPHER` — parser errors on it with + guidance to move the cipher into the pair. +* Boot logs each pair's fingerprint (`sha256(key)[..12]` for real ciphers, + `—` for `none`) and marks the head. Warn line if head fp differs from any + non-head fp (rotation window signal). +* Docs: `example.env`, `docs/config/env.md`, `docs/guide/backend-storage.md` + updated with the pair syntax and the three recipes (rotate / encrypt / + decrypt). + +**Exit criteria.** Boot with a 1-pair config behaves identically to today. +Boot with a 2-pair config succeeds and logs both fingerprints. Boot with +`_ENCRYPTION_CIPHER` alongside `_ENCRYPTION_KEY` fails with a migration hint. + +### Slice K2 — v1 read/write paths in `EncryptedBlobBackend` + +**Scope.** The encrypted-backend decorator learns the v1 header format. Reads +dispatch on magic bytes; writes always emit v1 headers. Legacy read path +preserved verbatim. `blobs_consistency` gains a magic-byte branch to track +the legacy-blob count. + +* `EncryptedBlobBackend` refactored around `BlobFormat::V1` writer + reader. + Legacy path preserved but read-only from new code. +* Read dispatches on magic; write always emits v1 header at `.blob`. +* `blobs_consistency`: during its normal walk, per-blob magic-byte check; + legacy count recorded on the run's `stats` JSON. No schema hit — reuses + the existing `stats` bag on `jobs.recoverable_runs`. +* `storage_migrate`: no changes needed. It copies raw bytes between backends; + format is preserved on the target automatically because the object bytes + are opaque to it. + +**Exit criteria.** A brand-new deployment writes only v1 blobs. An upgraded +deployment reads existing legacy blobs and writes new v1 blobs at the same +object-key. `blobs_consistency` reports a legacy-blob count in its run stats. + +### Slice K3 — The `backend_rotate` recoverable job + +**Scope.** New handler, admin-triggered, iterates blobs, per-blob decision +tree (legacy → v1 upgrade, v1 with old key → v1 with head key, plaintext ↔ +encrypted where applicable), records findings. + +* New file `src/infrastructure/services/backend_rotate_service.rs`. +* Registered in `JobRegistry` as `backend_rotate`. Runs on the same + recoverable-runs engine (crash recovery, cursor persistence, pause/resume). +* Trigger endpoint: `POST /api/admin/storage/entries/{name}/rotate`. + Requires admin. Refuses if no work would happen (all blobs already at + head format + head key). Refuses if a `backend_rotate` or + `backend_migration` run is already Active for any entry. +* Per-blob decision tree per *The rotation job* section above. In-place + atomic replace at the same `.blob` object key. +* No readonly mode engaged. `X-Server-Status` header payload gains a + `rotation` field alongside `migration`. + +**Exit criteria.** On a filled sandbox: (1) legacy-only entry rotates to +v1; (2) encrypted entry with 2 pairs rotates so head-pair-only remains +decryptable; (3) plaintext entry with `none,cipher` head rotates to +encrypted-v1; (4) encrypted entry with `cipher,none` head rotates to +plaintext-v1. Each round-trip verified by dropping the retired pair from +config and successfully reading every blob. + +### Slice K4 — Admin panel action + banner + docs + +**Scope.** UI wiring, banner variant, guide docs, legacy-blob counter. + +* Entry card: **Rotate encryption key** button + tooltip states as above. +* Legacy-blob counter chip on each entry card. Sources the count from the + most recent `blobs_consistency` run's stats (already surfaced in the Jobs + admin page). Refresh button triggers a fresh scan. +* `ReadOnlyBanner.svelte` gains a third variant `variant="rotating"` — same + shape, milder tone (info not warning), copy makes it clear that operations + continue. +* Server-status header payload gains `rotation?: {entry, migrated, total, + percent}` alongside the existing `migration?:` field. `readonly` stays + false during rotation. +* `docs/guide/backend-storage.md`: new **Rotating an encryption key**, + **Encrypting a plaintext deployment**, and **Decrypting an encrypted + deployment** sections with the recipes above. +* `docs/config/env.md`: pair syntax section under `_ENCRYPTION_KEY`. +* Admin panel *"Rotation complete — safe to remove the old key"* hint gated + on findings=0 AND legacy count=0. + +**Exit criteria.** Round-trip on a real deployment (Ed's OVH S3): add pair +→ restart → rotate → verify blobs (including legacy) → remove old pair → +restart → blobs still readable. + +## Forward compatibility (v2 and beyond) + +The `` field is the single load-bearing knob for future format +changes. Reserved slots: + +* `0x0001` — this plan. AES-256-GCM, server-side keys, individual blob + integrity via AEAD tag + content-hash. +* `0x0002…0x00FF` — reserved for server-side format bumps: new AEAD, + chain-authenticated CDC chunks (either *manifest HMAC* — no header changes, + `chunk_manifests` gains a server-computed HMAC over the ordered chunk-hash + list — or *Merkle root in header* — v2 grows to carry `manifest_root` + + `chunk_index`, letting any streamed chunk be verified in isolation). Both + defend against manifest reorder / truncation / injection attacks that + content-addressability alone doesn't cover. +* `0x0100…0x01FF` — reserved for client-side encryption (E2E) variants. + `` becomes the client-key fingerprint; the server can no longer + decrypt and passes ciphertext through the streaming path unchanged. + +v1 and v2 coexist in the same storage indefinitely — the magic-byte read +dispatch handles arbitrary versions at position 5-6. Migration between +generations reuses `backend_rotate`'s pattern: rewrite each blob with the +new-generation writer, in-place at the same object key. + +## Non-goals + +* **Asymmetric / KMS-backed keys.** Out of scope. Symmetric AES-256-GCM only. + A KMS-backed variant is a separate slice built on top of this one. +* **Chunk-chain integrity / manifest tampering defense.** Recognised as a + real gap (chunk reorder / truncate / inject via DB write). Deferred to the + v2 header slice — see *Forward compatibility*. +* **Client-side encryption (E2E).** Same — v2 slot reserved, separate slice. +* **Automatic old-key removal.** Explicit human step, see above. +* **Rotation of a key on the "in-transit" side** (client → server TLS). + Handled by the reverse proxy; unrelated. +* **Per-blob key derivation** (KDF from a master key + blob hash). A + legitimate hardening path but orthogonal to the rotation story; folds in as + a future slice under this same pair-list config. +* **Rollback safety across format generations.** If v1 code is rolled back + to pre-v1 after new v1 blobs have been written, old code will 500 on those + blobs (AES-GCM tag fails on OXCPT-prefixed bytes). Not addressed by this + plan; the safety net is restore-from-backup. Deploying an + incompatibility-breaking format change is a pre-planned event, not a hot + rollback scenario. + +## Open questions + +* **Should we throttle the rotate job?** Same question `backend_migration` had. + Answer: not in v1. If throughput bites, add a `_ROTATE_MAX_MB_PER_SEC` on + the entry later. +* **Should rotation be idempotent under repeat trigger?** Yes. Running it a + second time with the same config is a no-op walk (every blob is already at + head format + head key). +* **Should we ship a `format_generation` column on `storage.blobs` as a + query-optimizer?** No in v1. Consistency-scan stats cover the "how many + legacy blobs remain" question at admin cadence, and per-read magic-byte + dispatch is a 5-byte compare. If a real hot-path or admin-panel latency + need emerges, the column is a small back-fill migration to add later. +* **Should the legacy-blob counter block key removal at code level?** No — + keeping the trust model consistent with "admin edits .env, we don't fight + them" is the current default. The counter + hint is enough. If real + incidents happen we can escalate to a hard guard later. diff --git a/docs/plan/storage-multi-entry.md b/docs/plan/storage-multi-entry.md index 584aab50..88bb9a6c 100644 --- a/docs/plan/storage-multi-entry.md +++ b/docs/plan/storage-multi-entry.md @@ -213,7 +213,7 @@ if permission.is_write() && self.migration_readonly.load(Ordering::Relaxed) { - **Reads are unaffected**. Users can still browse and download during migration. - **Boot-time clearing**: if boot detects `migration_readonly=true` AND no - in-flight `storage_migration` row (no `Running`/`Paused`) AND + in-flight `backend_migration` row (no `Running`/`Paused`) AND `active_backend_name` matches the entry the app booted onto → assume successful cutover completed on prior boot, clear the flag. Otherwise leave it set; admin knows they still need to finish something. @@ -230,7 +230,7 @@ if permission.is_write() && self.migration_readonly.load(Ordering::Relaxed) { strings — but the identity check still runs as a second-line defence against the encryption-in-place case) - Write admin_settings.storage.migration_readonly = true - - Trigger `storage_migration` recoverable job with + - Trigger `backend_migration` recoverable job with params = { source_name: "local_main", target_name: "s3_prod" } 3. Migration runs — target resolved fresh each batch by NAME lookup, so @@ -473,5 +473,5 @@ Per slice, plus these end-to-end scenarios in Hurl: `PgAclEngine::check_inner` is where write-permission short-circuits live; the new global read-only clause lands next to the per-drive one. - `docs/plan/job-registry.md` Part 2 — recoverable-run engine that - `storage_migration` runs on; `params` field, resume semantics, boot + `backend_migration` runs on; `params` field, resume semantics, boot sweep. diff --git a/example.env b/example.env index f84b531a..45b9b06e 100644 --- a/example.env +++ b/example.env @@ -371,9 +371,16 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud # `OXICLOUD_STORAGE__BACKEND` (local | s3 | azure) plus the # backend-specific fields below. A missing required field aborts # boot with the exact var name in the error message. -# * Presence of `OXICLOUD_STORAGE__ENCRYPTION_KEY` implies AES-256 -# encryption is enabled on that entry (no separate enable flag). -# Bad base64 / wrong length aborts boot with the entry name. +# * `OXICLOUD_STORAGE__ENCRYPTION_KEY` is a comma-separated LIST +# of `:` pairs. Presence implies encryption is +# enabled on that entry (no separate enable flag). The LAST pair +# wins on writes; every pair is a candidate for reads. Supported +# ciphers: `aes-256-gcm` (default when no cipher prefix is given) +# and `none` (empty-key sentinel used at pair-list head for a +# decrypt-in-place rotation, or at tail for an encrypt-in-place +# rotation). Bad base64 / wrong length / duplicate keys / +# multiple `none` pairs abort boot with the entry name. See +# `docs/plan/storage-key-rotation.md` for rotation recipes. # * SETTING `_ENTRIES` alongside the legacy flat vars below (e.g. # `OXICLOUD_STORAGE_BACKEND` + `OXICLOUD_S3_BUCKET`) is a FAIL-FAST # boot error — pick one mode. Migrate any leftover flat vars into @@ -393,11 +400,14 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud #OXICLOUD_STORAGE_s3_prod_S3_ACCESS_KEY= #OXICLOUD_STORAGE_s3_prod_S3_SECRET_KEY= #OXICLOUD_STORAGE_s3_prod_S3_FORCE_PATH_STYLE=false -#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY= # generate: openssl rand -base64 32 -# Cipher declaration — future-proofing. Today only `aes-256-gcm` is -# accepted (and it's the default when `_ENCRYPTION_KEY` is set), so -# this line can be omitted. Explicit here as documentation. -#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_CIPHER=aes-256-gcm +#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm: # generate: openssl rand -base64 32 +# The bare shorthand `` (no `aes-256-gcm:` prefix) +# also works; the explicit form makes the cipher visible and is +# required once you have more than one pair in the list. +# +# Two-pair rotation example (paste both pairs in .env, restart, +# then trigger the format-upgrade job, then drop the OLD pair): +#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=aes-256-gcm:,aes-256-gcm: # # Repair flag: if you rename an entry in .env while the DB still points # at the old name, boot aborts with an actionable error pointing at: diff --git a/examples/bench_s3_put.rs b/examples/bench_s3_put.rs index 19902152..3e32f416 100644 --- a/examples/bench_s3_put.rs +++ b/examples/bench_s3_put.rs @@ -353,7 +353,7 @@ async fn main() { // Full production composition: Cache(Encrypted(Retry(S3))). let cache_dir_b = tempfile::tempdir().expect("tempdir"); let full_stack: Arc = Arc::new(CachedBlobBackend::new( - Arc::new(EncryptedBlobBackend::new( + Arc::new(EncryptedBlobBackend::new_single_aes( Arc::new(RetryBlobBackend::new( backend.clone() as Arc, RetryPolicy::default(), diff --git a/frontend/src/lib/api/endpoints/admin.ts b/frontend/src/lib/api/endpoints/admin.ts index a4001438..23834d19 100644 --- a/frontend/src/lib/api/endpoints/admin.ts +++ b/frontend/src/lib/api/endpoints/admin.ts @@ -65,9 +65,13 @@ export function reextractPhotoMetadata(): Promise { } /** A freshly generated AES-256 at-rest blob-encryption key (base64) plus a - * data-loss warning authored by the server. */ + * data-loss warning authored by the server. The `fingerprint` is the + * SSH-style colon-hex render the boot log / admin pair-chain / rotate + * reports all use — admins can paste the key into `.env`, restart, and + * check the fingerprint matches to confirm the key made it in intact. */ export interface GeneratedKey { key: string; + fingerprint: string; warning: string; } @@ -338,20 +342,32 @@ export function promoteUserToInternal(userId: string): Promise { // ── Dashboard ─────────────────────────────────────────────────────────── +export interface DriveKindUsage { + kind: 'personal' | 'shared'; + used_bytes: number; + // null when there are no capped drives of this kind — the FE + // hides the ratio and just renders "N unlimited" + capped_quota_bytes: number | null; + unlimited_count: number; + capped_count: number; +} + export interface AdminDashboard { total_users: number; active_users: number; admin_users: number; server_version: string; - total_used_bytes: number; - total_quota_bytes: number; - storage_usage_percent: number; + drive_usage: DriveKindUsage[]; auth_enabled: boolean; oidc_configured: boolean; quotas_enabled: boolean; registration_enabled?: boolean; users_over_80_percent: number; users_over_quota: number; + // Backend physical accounting — omitted when the dedup service + // is unavailable. Renders as "—" in that case. + total_bytes_stored?: number; + dedup_ratio?: number; } export function getDashboard(): Promise { @@ -466,6 +482,25 @@ export function saveOidc(body: Record): Promise { // ── Storage settings + migration ─────────────────────────────────────────── +/** + * One `:` pair rendered for the admin storage panel. + * Never carries key material — only cipher name + SSH-style + * fingerprint safe to show operators. + */ +export interface StorageEncryptionPair { + /** `"aes-256-gcm"` for a real-cipher pair, `"none"` for a `none:` sentinel. */ + cipher: string; + /** + * SSH-style colon-hex 8-byte fingerprint of the key. Matches + * `backend_rotate`'s `head_key_fp` and the `oxicloud --fingerprint` + * CLI output — enables one-glance identification of which key is + * which. `undefined` for `none:` pairs (no key material). + */ + fingerprint?: string; + /** True for the LAST pair in the list — the write pair (head). */ + is_head: boolean; +} + export interface StorageEntrySummary { name: string; backend: string; @@ -473,6 +508,14 @@ export interface StorageEntrySummary { encryption_enabled: boolean; /** Human-readable physical hint (root_dir / bucket / container). */ location_hint?: string | null; + /** + * Ordered pair-list summary — oldest first, head last. Empty when + * the entry has no `_ENCRYPTION_KEY` declared at all. Used by the + * entry card to render the pair chain so admins can identify + * which key is the current head + which are safe to remove after + * a completed rotation. + */ + encryption_pairs: StorageEncryptionPair[]; } export interface StorageSettings { @@ -538,6 +581,22 @@ export function migrationAction( return mutate(`/api/admin/storage/migration/${action}`, 'POST', body); } +/** + * K4 (storage-key-rotation): trigger `backend_rotate` on a specific + * storage entry. Normalises every blob on `` to that entry's + * head-pair format: legacy → v1, plaintext ↔ encrypted, old-key → + * new-key. Fire-and-forget — poll `GET /api/admin/jobs/backend_rotate` + * for status. + * + * Backend: `POST /api/admin/storage/entries/{name}/rotate` + * (`admin_handler::trigger_backend_rotate`). Refuses (400) on unknown + * entry name or when a `backend_rotate` / `backend_migration` run is + * already in flight. + */ +export function rotateStorageEntry(name: string): Promise { + return mutate(`/api/admin/storage/entries/${encodeURIComponent(name)}/rotate`, 'POST', undefined); +} + // verifyMigration + MigrationVerifyResult retired in slice 7 of // docs/plan/storage-multi-entry.md — the corresponding backend // endpoint's sample-based check is superseded by diff --git a/frontend/src/lib/api/endpoints/adminJobs.ts b/frontend/src/lib/api/endpoints/adminJobs.ts index 97db6419..71bf7d65 100644 --- a/frontend/src/lib/api/endpoints/adminJobs.ts +++ b/frontend/src/lib/api/endpoints/adminJobs.ts @@ -17,18 +17,37 @@ const JSON_HEADERS = { 'Content-Type': 'application/json' }; * `ok: true` means "dispatch reached the handler"; the handler's own * pass/fail is in `outcome.outcome`. For `consistency_batch`, per-child * outcomes are inside `outcome.extra.per_check`. + * + * `outcome` is absent for detached jobs (currently only + * `backend_migration`) — the endpoint returns `202 Accepted` with + * `dispatched: true` immediately and the run continues in the + * background. Progress polling shows the state; there's no synchronous + * outcome to surface. */ export interface TriggerResponse { ok: boolean; - outcome: JobOutcome; + outcome?: JobOutcome; + dispatched?: boolean; + detached?: boolean; } -/** Envelope from `POST /api/admin/jobs/{name}/cancel`. `run_id` is - * the id of the run whose `Running` status was flipped to - * `CancelRequested` (null when nothing was in flight to cancel). */ +/** Envelope from `POST /api/admin/jobs/{name}/cancel` — terminal + * cancel. `run_id` populated iff a non-terminal row was flipped + * (Running/CancelRequested get the intent stamp; Paused gets a + * direct DB flip to Cancelled). */ export interface CancelResponse { - ok: boolean; - run_id: string | null; + cancelled: boolean; + run_id?: string; + reason?: string; + note?: string; +} + +/** Envelope from `POST /api/admin/jobs/{name}/pause` — soft pause. */ +export interface PauseResponse { + paused: boolean; + run_id?: string; + reason?: string; + note?: string; } /** @@ -84,11 +103,11 @@ export async function triggerJob( } /** - * `POST /api/admin/jobs/{name}/cancel` — cooperatively request cancel - * of the currently running instance. The handler observes it on its - * next `store.status()` poll and returns `RunOutcome::Paused` at the - * next safe boundary. If nothing is running, this is a no-op that - * returns `run_id: null`. + * `POST /api/admin/jobs/{name}/cancel` — TERMINAL cancel. Abandons + * the run: Running/CancelRequested rows get stamped with the intent + * flag and land as `Cancelled` when the handler yields; Paused rows + * get flipped directly to `Cancelled`. Not resumable. Use `pauseJob` + * for interruption-with-resume semantics. */ export async function cancelJob(name: string): Promise { const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/cancel`, { @@ -109,6 +128,31 @@ export async function cancelJob(name: string): Promise { return (await res.json()) as CancelResponse; } +/** + * `POST /api/admin/jobs/{name}/pause` — cooperative pause. Row lands + * as `Paused` when the handler yields; a subsequent trigger click + * resumes from the cursor via `run_or_resume`. Use `cancelJob` to + * abandon terminally. + */ +export async function pauseJob(name: string): Promise { + const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/pause`, { + method: 'POST', + credentials: 'same-origin', + headers: { ...JSON_HEADERS, ...getCsrfHeaders() } + }); + if (!res.ok) { + let msg = `pause failed: ${res.status}`; + try { + const body = (await res.json()) as { error?: string; message?: string }; + msg = body.error ?? body.message ?? msg; + } catch { + /* no JSON body */ + } + throw new Error(msg); + } + return (await res.json()) as PauseResponse; +} + /** * `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable * runs for `name`, newest first. Backend caps `limit` at 100. diff --git a/frontend/src/lib/api/types.ts b/frontend/src/lib/api/types.ts index c23efc17..f6db9397 100644 --- a/frontend/src/lib/api/types.ts +++ b/frontend/src/lib/api/types.ts @@ -543,6 +543,19 @@ export type JobOutcome = * Cadence + last-run bookkeeping. `interval_ms` / `next_run_at` are * `undefined` on on-demand jobs (serde skips `Option::None`). */ +/** + * Enough info about a paused recoverable run for the admin panel to + * render "Resume (scanned/total)" on the job row without opening the + * drawer. Absent when no `Paused` row exists for this job. `total` + * is absent when the tenant didn't seed a countable subject — + * fallback UI is just "Resume". + */ +export interface PausedRunBrief { + id: string; + scanned: number; + total?: number; +} + export interface JobSummary { name: string; interval_ms?: number; @@ -550,6 +563,20 @@ export interface JobSummary { last_run_at?: string; last_outcome?: JobOutcome; running: boolean; + /** + * `true` iff the job persists runs + findings to + * `jobs.recoverable_runs`. Consumed by the admin panel to decide + * whether the row is expandable (drawer with run history + + * findings) and to gate the retention/purge action — replaces + * the pre-K3 name-based allowlist that missed newly-added + * recoverable tenants (`backend_rotate` shipped first without a + * row-expand until this flag was added). + */ + recoverable: boolean; + /** Populated iff a `Paused` row exists in `jobs.recoverable_runs` + * for this job. Distinct from `running` — a paused run is + * resumable via the same trigger endpoint. */ + paused_run?: PausedRunBrief; } /** @@ -557,7 +584,13 @@ export interface JobSummary { * non-terminal set (Running / Paused / CancelRequested) is what the * DB's `one_active_run_per_job` partial unique index scopes. */ -export type RunStatus = 'Running' | 'Paused' | 'CancelRequested' | 'Completed' | 'Failed'; +export type RunStatus = + | 'Running' + | 'Paused' + | 'CancelRequested' + | 'Completed' + | 'Failed' + | 'Cancelled'; /** * `RunSummary` — one row per recoverable-job run from diff --git a/frontend/src/lib/components/AdminJobsPanel.svelte b/frontend/src/lib/components/AdminJobsPanel.svelte index 0d55d949..2d5fd49e 100644 --- a/frontend/src/lib/components/AdminJobsPanel.svelte +++ b/frontend/src/lib/components/AdminJobsPanel.svelte @@ -28,6 +28,7 @@ listJobs, listRuns, listFindings, + pauseJob, triggerJob, cancelJob, purgeJobRuns @@ -237,11 +238,33 @@ const key = `trigger:${name}${opts.deep ? ':deep' : ''}`; markBusy(key, true); try { - const res = await triggerJob(name, opts); + // Fire the trigger + a follow-up loadJobs after a short delay + // in parallel. Long jobs (backend_migration) come back 202 + // immediately; short jobs (consistency checks) come back on + // completion. Either way, the `running` badge / Pause button + // should appear within a render cycle rather than waiting + // for the next 5s poll tick. + const triggerPromise = triggerJob(name, opts); + // Give the backend a moment to register the run's + // `current_run_start` before we ask "is it running?" — this + // races against the trigger acknowledgment for detached + // jobs. 300 ms is well under the 5 s poll cadence and + // invisible to the operator. + setTimeout(() => { + void loadJobs(); + if (expandedJob === name) void loadRuns(expandedJob); + }, 300); + + const res = await triggerPromise; // The trigger envelope carries the child's outcome — surface // its pass/fail immediately so operators don't have to click - // through to see whether the run completed cleanly. - if (res.outcome.outcome === 'ok') { + // through to see whether the run completed cleanly. Detached + // jobs come back with `dispatched: true` and no outcome — + // silence the notify for those (the "started" state is + // already visible via the badge). + if (!res.outcome) { + // dispatched (detached) — no outcome to render + } else if (res.outcome.outcome === 'ok') { ui.notify( t('admin.jobs.triggered_ok', { name }, '{{name}} triggered successfully'), 'success' @@ -265,23 +288,71 @@ } } - async function onCancel(name: string) { - const key = `cancel:${name}`; + async function onPause(name: string) { + const key = `pause:${name}`; markBusy(key, true); try { - const res = await cancelJob(name); - if (res.run_id) { + const res = await pauseJob(name); + if (res.paused) { ui.notify( t( - 'admin.jobs.cancel_requested', + 'admin.jobs.pause_requested', { name }, - 'Cancel requested — {{name}} will pause at the next safe boundary' + 'Pause requested — {{name}} will pause at the next checkpoint (progress preserved)' ), 'info' ); } else { ui.notify( - t('admin.jobs.cancel_noop', { name }, 'Nothing to cancel — {{name}} is not running'), + t('admin.jobs.pause_noop', { name }, 'Nothing to pause — {{name}} is not running'), + 'info' + ); + } + await loadJobs(); + if (expandedJob === name) await loadRuns(name); + } catch (e) { + ui.notify(errorMessage(e), 'error'); + } finally { + markBusy(key, false); + } + } + + async function onCancel(name: string) { + // Terminal cancel confirmation — this is destructive (marks the + // run as Cancelled, cursor preserved for post-mortem but not + // resumable). Skip the confirm for non-recoverable jobs since + // there's no persistent state to lose there today. + if ( + !window.confirm( + t( + 'admin.jobs.cancel_confirm', + { name }, + 'Cancel run of {{name}}? The run will be marked as Cancelled and cannot be resumed. Progress bytes on disk stay put — this only affects the run row.' + ) + ) + ) { + return; + } + const key = `cancel:${name}`; + markBusy(key, true); + try { + const res = await cancelJob(name); + if (res.cancelled) { + ui.notify( + t( + 'admin.jobs.cancel_requested', + { name }, + 'Cancel requested — {{name}} will land in Cancelled at the next batch boundary (Paused rows flip immediately)' + ), + 'info' + ); + } else { + ui.notify( + t( + 'admin.jobs.cancel_noop', + { name }, + 'Nothing to cancel — {{name}} has no non-terminal run' + ), 'info' ); } @@ -400,11 +471,41 @@ return 'jobs-panel__pill jobs-panel__pill--ok'; case 'Failed': return 'jobs-panel__pill jobs-panel__pill--err'; + case 'Cancelled': + return 'jobs-panel__pill jobs-panel__pill--neutral'; default: return 'jobs-panel__pill jobs-panel__pill--neutral'; } } + /** + * Human-facing label for a `RunStatus`. Translates the internal + * DB status enum into text an operator can read at a glance — + * notably renders `CancelRequested` as "Pausing" for the + * recoverable-run case (the mechanism is a cancel flag, but the + * user intent is pause). Non-recoverable cancels aren't a thing + * today because non-recoverable jobs run to completion inline, + * so `CancelRequested` here is always the pause path. + */ + function statusLabel(status: RunStatus): string { + switch (status) { + case 'Running': + return t('admin.jobs.status_running', 'Running'); + case 'Paused': + return t('admin.jobs.status_paused', 'Paused'); + case 'CancelRequested': + return t('admin.jobs.status_ending', 'Ending'); + case 'Completed': + return t('admin.jobs.status_completed', 'Completed'); + case 'Failed': + return t('admin.jobs.status_failed', 'Failed'); + case 'Cancelled': + return t('admin.jobs.status_cancelled', 'Cancelled'); + default: + return status; + } + } + /** Coarse "3 min ago" / "2 h ago" — same shape as the parent * admin page's timeAgo(). Duplicated locally so the component * stays self-contained; extract if a third caller emerges. */ @@ -461,16 +562,13 @@ } function isRecoverable(job: JobSummary): boolean { - // Heuristic: recoverable jobs are the ones that publish runs via - // `jobs.recoverable_runs`. There's no direct flag on JobSummary - // (Part 1 handlers and Part 2 adapters share the same summary - // shape by design). Name-based recognition is fine for now — the - // admin panel is the only consumer; broader use would call for - // a `recoverable: bool` field in JobSummary. - return name_is_recoverable(job.name); - } - function name_is_recoverable(name: string): boolean { - return name.endsWith('_consistency') || name === 'storage_migration'; + // Backend authoritative source: the `recoverable` flag on + // `JobSummary` is set at registration time by + // `RecoverableAdapter::is_recoverable() -> true`. Every tenant + // registered via `register_recoverable_job` flips it + // automatically. No name-based allowlists — a new recoverable + // tenant is expandable in the UI as soon as it's registered. + return job.recoverable; } // Consistency batch shortcut — top button. Only shown when the @@ -487,7 +585,6 @@
-

{t('admin.jobs.title', 'Jobs')}

{t( 'admin.jobs.hint', @@ -620,30 +717,94 @@ {/if} - - {#if supportsDeep(job.name)} + {#if job.paused_run} + + {@const p = job.paused_run} + {@const label = + p.total && p.total > 0 + ? t( + 'admin.jobs.resume_progress', + { scanned: p.scanned, total: p.total }, + 'Resume ({{scanned}}/{{total}})' + ) + : t('admin.jobs.resume', 'Resume')} - {/if} - {#if isRunning(job) && canExpand} + {:else} + + {#if supportsDeep(job.name)} + + {/if} + {/if} + {#if isRunning(job) && canExpand} + {#if isRecoverable(job)} + + + + {:else} + + {/if} {/if} @@ -710,7 +871,7 @@ - {run.status} + {statusLabel(run.status)} @@ -1009,10 +1170,6 @@ flex-wrap: wrap; } - .jobs-panel__header-text h2 { - margin: 0 0 0.25rem; - } - .jobs-panel__hint { margin: 0; color: var(--color-text-muted); @@ -1254,9 +1411,24 @@ background: var(--color-bg-surface); padding: 0.5rem; border-radius: 4px; - overflow-x: auto; font-size: 0.8rem; margin: 0.5rem 0 0; + /* Wrap long values (cursor_hex is 128 hex chars) instead of + expanding the table cell — the run-drawer sits inside a + `` that would otherwise grow horizontally past + the viewport and blow out the page layout. `pre-wrap` + preserves the multi-line JSON.stringify(…, 2) indent; + `word-break: break-all` breaks the long hex strings mid-run + without hyphens. + + `overflow-x: auto` is kept as a defense-in-depth for any + future field that pre-wrap can't handle (e.g. a single + unbroken word longer than max-width). It only kicks in + when wrapping isn't enough. */ + white-space: pre-wrap; + word-break: break-all; + max-width: 100%; + overflow-x: auto; } .jobs-panel__findings h4 { diff --git a/frontend/src/lib/components/AppShell.svelte b/frontend/src/lib/components/AppShell.svelte index d669b4dd..5dc8f067 100644 --- a/frontend/src/lib/components/AppShell.svelte +++ b/frontend/src/lib/components/AppShell.svelte @@ -1041,6 +1041,13 @@ different copy. --> {#if serverStatus().readonly} + {:else if serverStatus().rotation} + + {/if} {@render children()}

diff --git a/frontend/src/lib/components/ReadOnlyBanner.svelte b/frontend/src/lib/components/ReadOnlyBanner.svelte index d7fd5491..6a5aa61f 100644 --- a/frontend/src/lib/components/ReadOnlyBanner.svelte +++ b/frontend/src/lib/components/ReadOnlyBanner.svelte @@ -1,37 +1,27 @@