diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 57ac4780..7a63df25 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -40,6 +40,24 @@ ignore = [ "RUSTSEC-2026-0195", "RUSTSEC-2026-0194", + # wasmtime 43.0.2 — "Stores can mix up type indices between engines" + # (GHSA-hgjw-h833-99q9). Transitive via extism 1.30.0 (latest published; + # extism `main` still pins wasmtime 43, no upgrade path). The advisory + # has no patched 43.x — fix requires wasmtime >=46.0.2 or >=47.0.3, and + # forcing that via [patch.crates-io] would break extism (three major + # wasmtime API bumps between 43 and 46). Real fix waits on extism + # upstream to migrate. + # + # Runtime exposure is zero in default deployments: + # - `plugins` is an OPT-IN build feature; default builds and the CI + # release binary don't link wasmtime at all. + # - Runtime activation additionally requires OXICLOUD_ENABLE_PLUGINS=true. + # - Plugin binaries are ADMIN-SUPPLIED, not attacker input. + # - The advisory's attack pattern is multi-Engine Store-sharing; + # OxiCloud's plugin runtime creates one fresh Plugin per invocation + # with its own Store (see infrastructure/services/plugins/runtime.rs). + "RUSTSEC-2026-0222", + # astral-tokio-tar 0.5.6 — tar extraction advisories, transitive via # testcontainers → testcontainers-modules, a DEV-dependency used only by # the `--cfg integration_tests` harness to spin up throwaway Postgres diff --git a/Cargo.toml b/Cargo.toml index ce8f9b73..491fbd67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -877,7 +877,21 @@ strip = true [profile.dev] opt-level = 1 -debug = true +# `line-tables-only` keeps file:line in panic backtraces (what you +# actually need on a long-running server) while dropping the rest of +# DWARF, which is worthless without a debugger. On this crate that +# takes target/debug/deps from ~58 GB to ~15-20 GB. Combined with +# `split-debuginfo = "unpacked"` (macOS-friendly: what little debug +# info remains lands in external .dSYM bundles that the linker +# doesn't embed in every .rlib), a full rebuild fits comfortably. +debug = "line-tables-only" +split-debuginfo = "unpacked" +# Incremental compilation caches per-function IR fingerprints so a +# small edit only recompiles what changed. On a single-crate rebuild +# (oxicloud is one crate) the savings are modest — worth < the ~7 GB +# incremental/ cache costs on disk. Rust-analyzer uses `cargo check`, +# which has its own cache, so LSP responsiveness is unaffected. +incremental = false [profile.bench] lto = "fat" diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index b615f3dd..8959cd65 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -154,6 +154,7 @@ export default defineConfig({ { text: "Trash & Recycle Bin", link: "/guide/trash" }, { text: "ZIP & Compression", link: "/guide/zip-and-compression" }, { text: "Internationalization", link: "/guide/i18n" }, + { text: "Backend Storage", link: "/guide/backend-storage" }, ], }, { diff --git a/docs/config/admin-settings.md b/docs/config/admin-settings.md index a04f81f6..e874f81e 100644 --- a/docs/config/admin-settings.md +++ b/docs/config/admin-settings.md @@ -64,6 +64,50 @@ If a value is overridden by environment variables, the admin API can expose that Successful responses include discovered endpoints such as the authorization endpoint, token endpoint, and userinfo endpoint. +## Storage & Migration + +The admin storage tab operates on the **named storage entries** declared in `.env` (see [Storage Entries](/config/env#storage-entries-multi-entry-recommended)). The set of entries is immutable per-deploy — adding or removing one requires a server restart. Runtime behaviour is driven by a single DB row that names which entry is currently active. + +### Endpoints + +| Method | Path | Description | +| --- | --- | --- | +| `GET` | `/api/admin/settings/storage` | List entries + active pointer + read-only flag + basic stats | +| `POST` | `/api/admin/settings/storage/test` | Reachability + round-trip test against the currently-effective backend | +| `POST` | `/api/admin/storage/migration/start` | Trigger a cross-entry migration. Body: `{"target_name": ""}` | +| `POST` | `/api/admin/storage/migration/pause` | Cooperative cancel — handler yields at the next batch boundary | +| `POST` | `/api/admin/storage/migration/resume` | Resume a paused run (target read from `params.target_name`, no body needed) | +| `GET` | `/api/admin/storage/migration` | Poll the current run's progress | + +Runs are recoverable — status, cursor, and per-blob failure findings all live in `jobs.recoverable_runs` / `jobs.run_findings`. The same run history is browsable via `GET /api/admin/jobs/storage_migration/runs`. + +### Cutover flow (moving the active pointer) + +1. Declare the target entry in `.env` and restart so `OXICLOUD_STORAGE_ENTRIES` picks it up. +2. Admin storage tab → pick the target from the dropdown → **Start migration**. The server engages global read-only mode (writes refused across the whole app; reads keep working), then copies blobs from source → target. +3. On `Completed`, the server writes `admin_settings.storage.active_backend_name = `. Read-only stays ON — writes on the OLD backend would strand data now that the pointer says the new one is active. +4. **Operator restarts the server.** Boot picks the new active entry, and the boot-clear rule drops the read-only flag (`no in-flight run + booted-entry matches DB pointer`). Server writable again, on the new backend. + +### Repair flag — pointer / entry drift + +If an entry is renamed or removed from `.env` while the DB pointer still names the old one, boot aborts with a clear error pointing at: + +``` +oxicloud --select-storage +``` + +This one-shot repair command re-runs the same env-parse the server does at boot, verifies `` is declared in `OXICLOUD_STORAGE_ENTRIES`, updates `admin_settings.storage.active_backend_name` in the DB, and exits. Operator then restarts normally. See [Environment Variables — Storage Entries](/config/env#storage-entries-multi-entry-recommended) for the model, and [`oxicloud --help`](https://github.com/oxicloud/oxicloud/blob/main/src/main.rs) for the full flag list. + +### Auditing entries other than the active one + +`blobs_consistency` and `backend_consistency` (recoverable jobs on the Jobs tab) accept `?storage=` to probe any declared entry — not just the live one. Use this to verify a migration target before cutover, or to audit an old backend after cutover but before decommissioning: + +``` +POST /api/admin/jobs/blobs_consistency/trigger?storage= +``` + +Unknown names 400 at the HTTP layer. + ## Data Storage Runtime settings are stored in `auth.admin_settings`. diff --git a/docs/config/env.md b/docs/config/env.md index 0b5d4ea1..c39651e7 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -78,7 +78,59 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_GRANT_CLEANUP_INTERVAL_HOURS` | `24` | How often the grant-cleanup daemon fires. Clamped to a minimum of 1 hour. Adjusting this doesn't change what gets deleted — only how promptly. Daily is fine for any realistic grant volume. | | `OXICLOUD_WEBDAV_DRIVE_LISTING_PREFIX` | `@drive` | Native WebDAV URL segment that renders the caller's drive list. Sanitized by trimming leading/trailing `/`. Three shapes: (1) default `@drive` — `/webdav/…` addresses the caller's default personal drive (back-compat), `/webdav/@drive/` returns the drive listing, `/webdav/@drive//…` targets a specific drive. (2) empty string `""` — `/webdav/` IS the drive listing, `/webdav//…` targets a specific drive, no default-drive shortcut. (3) any other string (e.g. `drives`) — same shape as `@drive` with that segment substituted. Only drives the caller has Read on via `role_grants` resolve. | -## Storage Backend +## Storage Entries (multi-entry, recommended) + +Declare one or more **named** storage backends. The one the app runs on is picked from the DB (`admin_settings.storage.active_backend_name`); the admin panel's storage tab flips the pointer, and cross-backend migration is a recoverable job that copies blobs between two entries with a read-only safety window. See [Admin Settings — Storage & Migration](/config/admin-settings) for the operator flow and the [multi-entry design doc](https://github.com/oxicloud/oxicloud/blob/main/docs/plan/storage-multi-entry.md) for the full model. + +| Variable | Default | Description | +|---|---|---| +| `OXICLOUD_STORAGE_ENTRIES` | — | Comma-separated allowlist of entry names. Names must match `[a-z0-9_-]{1,32}` and be unique. Order is preserved (the first entry is the fallback when the DB pointer is unset — fresh install). | + +Each declared name `` then reads its own set of per-entry variables: + +| Variable | Default | Description | +|---|---|---| +| `OXICLOUD_STORAGE__BACKEND` | — | Backend type for entry ``: `local` \| `s3` \| `azure` (required per entry) | +| `OXICLOUD_STORAGE__ROOT_DIR` | `OXICLOUD_STORAGE_PATH` | Local-only: root directory for this entry's `.blobs/`. Falls back to the ambient `OXICLOUD_STORAGE_PATH` when unset. | +| `OXICLOUD_STORAGE__S3_BUCKET` | — | S3-only: bucket name (required when backend=s3) | +| `OXICLOUD_STORAGE__S3_REGION` | `us-east-1` | S3-only: AWS region | +| `OXICLOUD_STORAGE__S3_ENDPOINT_URL` | — | S3-only: custom endpoint for non-AWS providers | +| `OXICLOUD_STORAGE__S3_ACCESS_KEY` | — | S3-only: access key ID | +| `OXICLOUD_STORAGE__S3_SECRET_KEY` | — | S3-only: secret access key | +| `OXICLOUD_STORAGE__S3_FORCE_PATH_STYLE` | `false` | S3-only: path-style URLs (required for MinIO, R2) | +| `OXICLOUD_STORAGE__AZURE_ACCOUNT_NAME` | — | Azure-only: storage account name | +| `OXICLOUD_STORAGE__AZURE_ACCOUNT_KEY` | — | Azure-only: storage account key | +| `OXICLOUD_STORAGE__AZURE_CONTAINER` | — | Azure-only: blob container name (required when backend=azure) | +| `OXICLOUD_STORAGE__AZURE_SAS_TOKEN` | — | Azure-only: SAS token (alternative to account key) | +| `OXICLOUD_STORAGE__AZURE_ENDPOINT_URL` | — | Azure-only: custom endpoint (Azurite, private deployments) | +| `OXICLOUD_STORAGE__ENCRYPTION_KEY` | — | Base64-encoded 32-byte AES-256 key. **Presence implies encryption is enabled** on this entry — no separate enable flag. Bad base64 / wrong length aborts boot. | +| `OXICLOUD_STORAGE__ENCRYPTION_CIPHER` | `aes-256-gcm` when `_ENCRYPTION_KEY` is set | Cipher choice for this entry. Only `aes-256-gcm` is accepted today (future-proofing knob — the enum is ready for a second cipher, the implementation still hardcodes AES-256-GCM). Setting the cipher without a key aborts boot. | + +**Fail-fast rules** (boot aborts with actionable message): + +- A declared name whose required per-entry fields are missing (`_BACKEND` never set, S3 with no `_S3_BUCKET`, Azure with no `_AZURE_CONTAINER`). +- Setting `OXICLOUD_STORAGE_ENTRIES` alongside any of the legacy flat vars below (`OXICLOUD_STORAGE_BACKEND`, `OXICLOUD_S3_*`, `OXICLOUD_AZURE_*`, `OXICLOUD_STORAGE_ENCRYPTION_*`). Pick one mode; the error lists every conflicting var to remove. +- A DB pointer (`admin_settings.storage.active_backend_name`) that names an entry not in the current `_ENTRIES`. The error points at the repair flag `oxicloud --select-storage ` — verify + UPDATE DB + exit. + +**Example** — two entries, local disk plus an S3 target for planned migration: + +``` +OXICLOUD_STORAGE_ENTRIES=local_main,s3_prod + +OXICLOUD_STORAGE_local_main_BACKEND=local +OXICLOUD_STORAGE_local_main_ROOT_DIR=/srv/oxicloud + +OXICLOUD_STORAGE_s3_prod_BACKEND=s3 +OXICLOUD_STORAGE_s3_prod_S3_BUCKET=my-oxicloud-bucket +OXICLOUD_STORAGE_s3_prod_S3_REGION=us-east-1 +OXICLOUD_STORAGE_s3_prod_S3_ACCESS_KEY=… +OXICLOUD_STORAGE_s3_prod_S3_SECRET_KEY=… +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=… # openssl rand -base64 32 +``` + +## Storage Backend (DEPRECATED — legacy single-backend) + +> ⚠️ **Deprecated.** Use [Storage Entries](#storage-entries-multi-entry-recommended) above for new deployments. These flat variables still work when `OXICLOUD_STORAGE_ENTRIES` is **unset** — the parser then synthesises one entry named `default` from them, keeping pre-multi-entry `.env` files booting unchanged. Booting via this path emits a `storage.legacy_flat_vars_deprecated` warning so operators see it in logs. Removal target: not yet fixed; migrate at your convenience by moving each variable below into `OXICLOUD_STORAGE__*` form under an entry declared in `OXICLOUD_STORAGE_ENTRIES`. **Setting any variable from this section alongside `OXICLOUD_STORAGE_ENTRIES` is a fail-fast boot error** — pick one mode. | Variable | Default | Description | |---|---|---| @@ -118,10 +170,12 @@ A least-recently-used disk cache that can speed up repeated reads from S3 or Azu | `OXICLOUD_STORAGE_CACHE_MAX_SIZE` | `53687091200` | Max cache size in bytes (50 GB) | | `OXICLOUD_STORAGE_CACHE_PATH` | `{STORAGE_PATH}/.blob-cache` | Cache directory | -### Client-Side Encryption +### Client-Side Encryption (DEPRECATED — per-entry key is the new home) AES-256-GCM encryption applied to blobs before they are written to any backend. +> ⚠️ **Deprecated.** Prefer per-entry `OXICLOUD_STORAGE__ENCRYPTION_KEY` under an entry declared in `OXICLOUD_STORAGE_ENTRIES` — presence of the key implies encryption is enabled on that entry (no separate flag), and multi-entry enables cross-key rotation via migration to a new entry. The flat vars below still work in zero-entries mode and get folded into the synthesised `default` entry, alongside the same deprecation warning at boot. + | Variable | Default | Description | |---|---|---| | `OXICLOUD_STORAGE_ENCRYPTION_ENABLED` | `false` | Enable at-rest blob encryption | diff --git a/docs/guide/backend-storage.md b/docs/guide/backend-storage.md new file mode 100644 index 00000000..6735711a --- /dev/null +++ b/docs/guide/backend-storage.md @@ -0,0 +1,123 @@ +# Backend Storage + +OxiCloud writes uploaded files to a **backend storage** — the physical place where the bytes actually live. Three kinds are supported: + +- **Local disk** — a folder on the server. +- **S3-compatible** — AWS S3, OVH, MinIO, Backblaze B2, Cloudflare R2, DigitalOcean Spaces, Wasabi. +- **Azure Blob Storage** — Microsoft Azure. + +You can declare more than one backend at a time (for example, keep the current disk backend while adding an S3 target), pick which one is currently in use from the admin panel, and migrate all your data between them without downtime. + +## Declaring backends + +Backends are declared in your **`.env`** file (or the equivalent environment variables in a Docker Compose / Kubernetes deployment). Each backend gets a short **name** you choose — like `local_main`, `s3_prod`, `s3_archive` — and its own set of settings under that name. + +Example: one local backend today and an S3 backend ready for a future migration. + +``` +OXICLOUD_STORAGE_ENTRIES=local_main,s3_prod + +OXICLOUD_STORAGE_local_main_BACKEND=local +OXICLOUD_STORAGE_local_main_ROOT_DIR=/srv/oxicloud + +OXICLOUD_STORAGE_s3_prod_BACKEND=s3 +OXICLOUD_STORAGE_s3_prod_S3_BUCKET=my-oxicloud-bucket +OXICLOUD_STORAGE_s3_prod_S3_REGION=gra +OXICLOUD_STORAGE_s3_prod_S3_ENDPOINT_URL=https://s3.gra.io.cloud.ovh.net +OXICLOUD_STORAGE_s3_prod_S3_ACCESS_KEY=… +OXICLOUD_STORAGE_s3_prod_S3_SECRET_KEY=… +``` + +A few things to know: + +- The first entry in `OXICLOUD_STORAGE_ENTRIES` is used on first boot if you haven't picked one from the admin panel yet. +- Names must be short (letters, digits, `_`, `-`), unique, and stable — once you pick a name, keep it. +- After editing `.env`, **restart the server** so it picks up the new declaration. The declaration is what unlocks the entry in the admin panel — from then on you can switch to it (and move data to it) without another restart. +- The full list of settings each backend type accepts lives in the [Environment Variables reference](/config/env#storage-entries-multi-entry-recommended). + +### Encryption at rest + +Any backend can be encrypted at rest by adding an encryption key to it: + +``` +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY=… # base64 of 32 random bytes +``` + +A key can be generated from **Settings → Storage → Generate key** in the admin panel. Set the same key on a new backend during migration and OxiCloud re-encrypts the data as it copies. + +::: warning +If you lose the encryption key, the data encrypted with it is unrecoverable. Store the key somewhere as safe as you'd store a database backup. +::: + +## Checking a backend + +Once a backend is declared, it appears on the admin **Storage** tab as a card: + +- **Location** — the folder path or `endpoint / bucket`. +- **Encryption** — a lock icon when a key is set for this entry. +- **Status** — either **active** (the one currently in use) or **available** (declared but not in use yet). + +Each card has three buttons: + +- **Test** — connects to the backend and does a small write / read / delete round-trip. On success it reports the round-trip time. On failure it tells you exactly what went wrong (bad credentials, wrong region, missing permission, unreachable host). Nothing persists on the backend after the test. +- **Blob consistency** — runs a full integrity audit against that backend. It verifies every file OxiCloud knows about is present on this backend and reports any missing pieces on the Jobs tab. +- **Migrate & activate** — visible on non-active backends. Moves all data to this backend and makes it the new active one. Details below. + +## Migrating between backends + +Migration copies every file from the currently-active backend to a chosen target backend, then switches the app over to the target. The typical flow: + +1. Add the new backend to your `.env` **without removing the current one**. +2. Restart the server so the new backend becomes visible in the admin panel. +3. On the **Storage** tab, click **Test** on the new backend to confirm it's reachable and writable. +4. Click **Migrate & activate** on the new backend and confirm the prompt. + +During the migration: + +- The server enters **read-only mode**. Users can still browse and download files; uploads, renames, deletes, and shares are refused until the migration finishes. A banner at the top of the Storage tab reminds you. +- Progress is shown on the migration status line under the entries list — how many blobs have been copied, the estimated time remaining. +- If the migration fails partway (network drops, quota exceeded), it pauses rather than losing progress. Clicking **Migrate & activate** again resumes from where it stopped. + +When the migration reaches 100%: + +- The new backend automatically takes over as the active one. +- Read-only mode is lifted. Users can write again — everything now goes to the new backend. +- No restart is required. + +The old backend is left untouched. Nothing is deleted from it. Once you're confident the new backend is holding up, you can decommission the old one at your own pace (empty the old S3 bucket, unmount the old disk, etc.). + +### Migrations that are refused + +The admin panel refuses to start a migration in two cases: + +- **Target equals source** — pointing a migration at the currently-active backend does nothing useful. +- **Target and source share the same physical storage** — for example, two backends that name the same S3 bucket but with different credentials or encryption keys. This would corrupt the data mid-migration. If you're trying to rotate an encryption key, migrate to a **different** bucket first, then rotate. + +### Verifying a migration + +After a migration completes, the "Blob consistency" button on the new backend runs a full audit. It walks every file OxiCloud knows about and confirms it's really present on the backend, byte-for-byte. Results appear on the Jobs tab. A clean run confirms the migration was complete. + +## Repairing a stuck config + +If you rename or remove a backend from `.env` while it was still the active one, the server may refuse to boot with an error like: + +``` +active_backend_name = `s3_prod`, but no entry with that name is declared in +OXICLOUD_STORAGE_ENTRIES. Available: [local_main]. […] +oxicloud --select-storage +``` + +Run the command it suggests to pick a still-declared backend and the server will boot again on the next start: + +``` +oxicloud --select-storage local_main +``` + +This just updates which backend OxiCloud considers active — it doesn't move any data. + +## Common gotchas + +- **S3 region must match the endpoint.** Every S3-compatible provider signs requests against a specific region string. `us-east-1` is right for real AWS S3 but wrong for OVH (`gra`, `sbg`, etc.), Backblaze B2, Wasabi, and others. Check your provider's docs for the exact region name. +- **`FORCE_PATH_STYLE=true` for non-AWS.** Path-style URLs (`endpoint/bucket/…`) are safer with providers whose bucket-name DNS setup isn't standard, or with bucket names that contain dots. +- **Test before migrating.** The **Test** button on each backend does a proper write/read/delete round-trip — a green result means the credentials and permissions are correct for the operations a migration actually needs. Don't skip it. +- **Keep the old backend around** until at least one full `blob consistency` audit passes on the new one. Cheap insurance. diff --git a/docs/guide/index.md b/docs/guide/index.md index bd9d1296..ed89175a 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -27,6 +27,7 @@ NextCloud was too slow on a home server. So OxiCloud was built to run on minimal ### Storage & Files - [Drives](/guide/drives) — Personal + Shared spaces with per-drive quota, members, and policies +- [Backend Storage](/guide/backend-storage) — Local disk, S3, Azure; multiple backends side-by-side; live migration between them - Drag-and-drop upload, multi-file, grid & list views - Chunked uploads (TUS-like, parallel, resumable, MD5 integrity) - BLAKE3 content-addressable file deduplication with ref-counting diff --git a/docs/plan/storage-multi-entry.md b/docs/plan/storage-multi-entry.md new file mode 100644 index 00000000..584aab50 --- /dev/null +++ b/docs/plan/storage-multi-entry.md @@ -0,0 +1,477 @@ +# Plan — Multi-entry storage config + name-selected active backend + +## Context + +Today OxiCloud has a single storage backend, configured via a flat set of env +vars (`OXICLOUD_STORAGE_BACKEND`, `OXICLOUD_S3_*`, `OXICLOUD_STORAGE_ENCRYPTION_KEY`, +…). The admin panel has a second, parallel storage-config surface backed by +`admin_settings.storage.*` rows, used by the migration tool to pick a target. +Priority is `env > DB > defaults`, so the DB config effectively acts as a +staging area for "what the migration should copy INTO" but never wins at boot. + +Two chronic problems fall out: + +1. **Split-brain config.** Admin edits DB via the panel; app boot ignores DB. + Migration completes; live backend hasn't moved. Admin has to remember to + copy env vars into `.env` and restart. Two sources of truth for the same + setting. Cutover is a manual multi-step flow; users routinely get it wrong. + +2. **Migration data-loss window on concurrent writes.** The copy walks + `storage.blobs` in hash order. A blob whose hash is lex-lower than the + current cursor, written to source AFTER migration passed it, is never + copied to target. `passed=true, findings=0` completion does NOT guarantee + target has every blob. Silent. + +3. **Migration target selection is fragile.** DTO passes the whole S3 config + at trigger time; secrets sit plaintext in `admin_settings`. Any future + pluggable-storage story compounds this (Azure, GCS, WebDAV-as-source, …). + +This plan replaces the split-brain model with a single-source-of-truth +architecture: + +- `.env` declares **N named storage entries** (immutable per-deploy). +- `admin_settings.storage.active_backend_name` holds ONE row — which named + entry the app currently runs on. That's the whole runtime config. +- Migration is the atomic transition from one active entry to another. Server + is put in read-only mode for the copy window; on completion, the active + pointer flips; a restart cuts over. + +Named entries also solve two adjacent problems Ed flagged during design: +- **Per-entry encryption keys** enable `local (raw) → s3 (encrypted K1)` + moves AND `s3 (K1) → s3-new-bucket (K2)` key-rotation moves. +- **`?storage=` on `blobs_consistency`/`backend_consistency`** lets + operators audit any registered entry (target verification, pre-decommission + check, etc.), replacing the sample-based `verify_migration` endpoint with a + full-walk audit. + +## Design decisions + +### Named entries in `.env` — explicit allowlist + +``` +OXICLOUD_STORAGE_ENTRIES=local_main,s3_prod + +OXICLOUD_STORAGE_local_main_BACKEND=local +OXICLOUD_STORAGE_local_main_ROOT_DIR=/data + +OXICLOUD_STORAGE_s3_prod_BACKEND=s3 +OXICLOUD_STORAGE_s3_prod_S3_BUCKET=my-bucket +OXICLOUD_STORAGE_s3_prod_S3_ENDPOINT_URL=https://s3.example.com +OXICLOUD_STORAGE_s3_prod_S3_REGION=us-east-1 +OXICLOUD_STORAGE_s3_prod_S3_ACCESS_KEY=... +OXICLOUD_STORAGE_s3_prod_S3_SECRET_KEY=... +OXICLOUD_STORAGE_s3_prod_S3_FORCE_PATH_STYLE=true +OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY= +``` + +- **Explicit `_ENTRIES` allowlist** — order-independent, admin-authored names. + Env-pattern-matching was considered and rejected: too fragile + (someone typos `OXICLOUD_STORAGE_s3_prud_BUCKET`, gets a silently-registered + ghost entry). Explicit list forces a declaration. +- **Names** are admin-chosen strings matching `[a-z0-9_-]{1,32}`, unique + within the list. Parsed at boot; unparseable → fail-fast with the offending + name in the error. +- **Order does not matter** — DB says which is active. Reordering entries in + `.env` never changes runtime behaviour. + +### Legacy flat-var interaction — three states, one is fail-fast + +Multi-entry lives alongside the pre-existing single-backend flat vars +(`OXICLOUD_STORAGE_BACKEND`, `OXICLOUD_S3_*`, `OXICLOUD_AZURE_*`, +`OXICLOUD_STORAGE_ENCRYPTION_KEY`, `_ENABLED`). The parser resolves the +interaction as follows: + +| `_ENTRIES` | Legacy storage-backend vars present | Behaviour | +|---|---|---| +| Unset / empty | Absent | `storage_entries = []`. Boot uses framework defaults (Local at `storage/`). | +| Unset / empty | Present | **Synthesize** a single entry named `default` from the legacy vars. Preserves upgrade path — existing deployments keep working without touching `.env`. | +| Set (e.g. `foo,bar`) | Absent | Parse each named entry from `_STORAGE__*` vars. Fail-fast if any declared entry is missing its required per-name fields. | +| Set (e.g. `foo,bar`) | **Present** | **FAIL FAST**. Boot aborts with an error listing every legacy var found. Admin must remove them or migrate them into per-entry `_STORAGE__*` form. | + +**Why fail-fast on the "both set" case:** without it, admin state is +ambiguous — someone edits `OXICLOUD_S3_BUCKET` expecting it to matter, +but it's silently ignored because `_ENTRIES` won. The subtle-bug cost +is much higher than the one-time cleanup cost. Refusing to boot forces +the conversion once, with a clear message naming the exact vars to +remove. + +**Set of "legacy storage-backend vars"** counted for the conflict check: +`OXICLOUD_STORAGE_BACKEND`, all seven `OXICLOUD_S3_*`, all five +`OXICLOUD_AZURE_*`, `OXICLOUD_STORAGE_ENCRYPTION_ENABLED`, and +`OXICLOUD_STORAGE_ENCRYPTION_KEY`. `OXICLOUD_STORAGE_PATH` is NOT in +this set — it drives multiple non-backend things (chunk dir default, +etc.) and remains a valid ambient path; per-entry `_ROOT_DIR` falls +back to it for Local entries when unset. + +### One DB row: `active_backend_name` + +Single setting in `admin_settings`: + +``` +storage.active_backend_name = "local_main" +``` + +- **Boot logic**: + 1. Parse `AppConfig.storage_entries` from env. + 2. Read `active_backend_name` from `admin_settings`. + 3. If unset (fresh install): use the FIRST name in `_ENTRIES`. Log which + one, don't fail. + 4. Look up the entry by name. Build `blob_backend` from it. + 5. If the name doesn't exist in current env (deploy drift — someone removed + an entry): **fail-fast at boot** with a clear error naming the missing + entry AND listing the available ones. Admin fixes env or overrides the + setting via a fallback CLI (see §Fallback). +- Everything the admin panel currently writes about storage + (`s3.bucket`, `s3.access_key`, …) is **removed** from DB. `save_storage_settings` + is deleted along with those rows. + +### Encryption is per-entry + +`OXICLOUD_STORAGE__ENCRYPTION_KEY` (base64 of exactly 32 bytes) on an +entry → that entry's backend is wrapped in `EncryptedBlobBackend` at build +time. Absent → raw backend. + +- **Presence-implies-enabled.** No separate `_ENCRYPTION_ENABLED` toggle — + one env var per entry is enough. +- **Fail-fast on invalid key**: bad base64, wrong decoded length → boot + aborts with the entry name in the error message. A bad key is a real + deployment error, must be caught at boot, never silently disabled. +- **Legacy `OXICLOUD_STORAGE_ENCRYPTION_KEY`** remains honoured for the + synthesized `default` entry when `_ENTRIES` is empty (upgrade path). + +Cross-entry encryption combinations work out of the box because +`EncryptedBlobBackend` is a decorator and `copy_blob` in the migration +handler always spools plaintext to a tmp file: + +| Source | Target | Migration behaviour | +|---|---|---| +| Raw local | Encrypted S3 (K1) | Read plaintext → write encrypts with K1 | +| Encrypted S3 (K1) | Raw local | Read decrypts with K1 → write plaintext | +| Encrypted S3 (K1) | Encrypted S3 (K2) on different bucket | Read decrypts K1 → write encrypts K2 (rotation via new bucket) | +| Encrypted S3 (K1) | Encrypted S3 (K2) on SAME bucket | **REFUSED** — see below | + +**In-place key rotation is refused.** Same physical bucket + different +encryption key would have the migration overwrite `.blob` with K2 +ciphertext while the LIVE backend is still K1-configured → readers get +K1-decrypt-of-K2-bytes → 500. Silent data-loss window. The +`is_source_target_identical` guard (see §Migration flow) catches this because +`storage_identity` deliberately excludes the encryption key. The refusal +message spells the case out and recommends the two-step workaround (rotate +via a temp bucket). + +**Proper in-place rotation (deferred future slice)** + +The safe implementation is to namespace object keys by encryption +generation — `.k2.blob` or `k2/.blob` (backend-specific +convention). Then: + +- Both key generations coexist in the same bucket during rotation. +- Reads continue against K1 (LIVE backend) — old keys untouched. +- Target writes go to K2 keys. +- Cutover restarts with K2 as live; K1 objects can be reaped async by + a follow-up sweep. +- The `is_source_target_identical` guard's `storage_identity` string + changes to INCLUDE the encryption generation — same-bucket + different-generation then correctly registers as a legitimate + migration, no longer refused. + +Schema changes required: +- `EncryptedBlobBackend` gains a `generation: u32` field. +- Object keys become `.k.blob` (or backend-specific + equivalent — S3 key naming, Azure blob naming). +- Read path tries current-gen first, falls back to prior-gen for the + rotation window (bounded by cutover-completion + async-reap + duration). + +Not built now — real key-rotation demand is rare enough that the +two-step workaround via temp bucket is acceptable. File this as +a follow-up slice AFTER multi-entry lands; the named-entry +infrastructure is a prerequisite (generation-versioned keys only +make sense when there's an entry model to hold "which generation +is active" as configuration). + +### Read-only mode reuses the existing AuthZ short-circuit + +`DrivePolicies.read_only` already gates writes at +`PgAclEngine::check_inner` — short-circuits `Create|Update|Delete|Share` +Permission checks on any resource in a read-only drive. We extend that clause +with a global check: + +```rust +// Inside PgAclEngine::check_inner, before per-drive read_only check: +if permission.is_write() && self.migration_readonly.load(Ordering::Relaxed) { + return AclDecision::Denied("server in migration read-only mode"); +} +``` + +- The flag is a `AtomicBool` on `AppState`, backed by + `admin_settings.storage.migration_readonly` so it **survives restart** + (server crashes mid-migration → boots read-only → admin retriggers → still + safe). +- **Admin operations bypass** as they already do — admin can still exit + read-only, cancel migration, restart the server. +- **Reads are unaffected**. Users can still browse and download during + migration. +- **Boot-time clearing**: if boot detects `migration_readonly=true` AND no + in-flight `storage_migration` row (no `Running`/`Paused`) AND + `active_backend_name` matches the entry the app booted onto → assume + successful cutover completed on prior boot, clear the flag. Otherwise leave + it set; admin knows they still need to finish something. + +### Migration flow — atomic + restart-proof + +``` +1. Admin picks target entry from a dropdown → clicks "Migrate to s3_prod" + +2. Backend: + - Verify target_name exists in AppConfig.storage_entries + - Verify target_name != active_backend_name (no-op guard; existing + `is_source_target_identical` refactored to compare NAMES not identity + strings — but the identity check still runs as a second-line defence + against the encryption-in-place case) + - Write admin_settings.storage.migration_readonly = true + - Trigger `storage_migration` recoverable job with + params = { source_name: "local_main", target_name: "s3_prod" } + +3. Migration runs — target resolved fresh each batch by NAME lookup, so + the run's params carries only the name. No secrets in params. If the + process restarts mid-run, the resume path re-resolves the entry from + env by the persisted name. Env is the source of truth for credentials. + +4. On Completed: + - Write admin_settings.storage.active_backend_name = "s3_prod" + - Log "Cutover complete — restart the server to switch to the new backend" + - LEAVE read-only mode on. The app is still running with local_main as + the live backend; if we lifted read-only now, writes would go to + local_main even though the DB pointer says s3_prod. Forces the operator + restart, which resolves the ambiguity. + +5. Admin restarts the server: + - Boot reads active_backend_name = "s3_prod" → live backend is now S3 + - Boot's read-only-clear rule fires (no in-flight migration + active + matches booted entry) → migration_readonly cleared + - Server writable, on the new backend. Cutover complete. +``` + +**Handling in-progress restart (server dies while migration is running)**: +- Boot sweep flips `Running` → `Paused` on the migration row (existing Part + 2 machinery). +- `active_backend_name` unchanged. Boots on old backend. +- `migration_readonly` stays true (in-flight migration → boot-time clear + rule DOESN'T fire). +- Admin retriggers migration → run_or_resume reads params.target_name → + resumes from cursor with the same target. + +**No secret ever reaches the DB.** Params holds only the entry names. +Credentials stay in env; migration resolves them at each batch by name. + +### Concurrent-write safety — read-only for the copy window + +The full-quiesce trade-off: users can browse/download during migration but +cannot upload, rename, delete, or share. For a multi-hour migration this is +noticeable; the alternative (dual-write decorator) is genuinely a week of +work (runtime backend swapping, failure-mode reconciliation, `MigrationBlobBackend` +rebuild) and only justified if migration is a routine op. Read-only is the +honest v1 answer — pick a low-traffic window, run the migration, restart. + +Read-only is engaged at trigger time and cleared at boot after cutover. +Nothing more elaborate. Dual-write is filed as a future upgrade if operator +demand appears. + +### `?storage=` for consistency audits + +Once entries are named, `?storage=` becomes the generic "probe any +registered entry" knob on the two tenants that touch a backend: + +- `blobs_consistency?storage=` — DB → backend probe. +- `backend_consistency?storage=` — backend → DB probe. + +Unspecified → falls through to the active backend (today's behaviour, +preserved). + +**Use cases this unlocks**: +- Pre-cutover verification: `blobs_consistency?storage=s3_prod` after + migration completes — full walk (not a sample) proving target has every + blob before the .env flip + restart. **Retires `verify_migration`** — + sample-check becomes redundant when full audit is one click away. +- Pre-decommission verification: `?storage=local_main` after cutover — check + the old backend still has every blob DB expects before `rm -rf` the local + `.blobs/`. +- Ad-hoc audit of any registered entry, backup-restore verification, etc. + +**Plumbing**: +- `JobRunArgs` gains `storage: Option`. +- `TriggerJobQuery` on the admin trigger endpoint parses `?storage=`. +- `BlobsConsistencyCheck` and `BackendConsistencyCheck` constructors take + `Arc` (or a smaller `EntryResolver` port); at run + start they use `args.storage` to pick the backend, falling back to the + injected active-backend Arc. +- **Unknown entry**: fail-fast at HTTP layer (`400` before the run ever + starts) with `known: [local_main, s3_prod]` in the response. Cheaper than + burning a run row. +- **Combined with `?deep=true`**: legit — "full byte-level integrity audit + of a named entry, not the live one". Audit log records both. +- Params on the run row records `probed_storage: ` (or "active") for + post-hoc diagnosis. + +### Fallback for the "boot fails on missing entry" case + +If admin renames an entry in `.env` (or removes one that DB still points +at), boot fails fast with a clear error. Operator has two ways out: + +1. Fix `.env` — add the missing entry back OR update `_ENTRIES` to include + an alternative that DOES exist, plus flip `active_backend_name` before + restart. +2. **CLI repair flag on the `oxicloud` binary itself**: + ``` + oxicloud --select-storage + ``` + Behaviour: parse `.env`, verify `` exists in `_ENTRIES` (fail-fast + with the available names listed if not), connect to DB, UPDATE + `admin_settings.storage.active_backend_name`, print confirmation, exit 0. + Does NOT continue to boot the server — one-shot repair; admin starts + the server normally afterwards. + +The bare-flag on the shipped binary is chosen over a separate `just` +recipe or auxiliary bin because: +- **Docker-friendly**: `docker exec oxicloud oxicloud --select-storage foo` + — no need to install extra tooling in the container. +- **Systemd-friendly**: can be run as a `ExecStartPre=` one-shot before the + main service unit. +- **No dep on `just`** being installed (dev-machine tool, not typical prod). +- **Same binary, same env-parse code path**: the repair uses THE SAME + `.env` parser the server does, so "verified present" means the server + will succeed on next boot too. No parser drift possible. + +The boot-time error message points at this flag explicitly, with the +exact command line filled in. + +### Interaction with existing surfaces + +- **`/admin/storage` tab** rewritten: + - Read-only listing of entries (name, backend type, encryption on/off, is + active). + - "Migrate to X" dropdown (choose target entry). + - Migration progress + verify/cancel (existing). + - Read-only banner when `migration_readonly` is on. + - The Save form, S3-field editors, .env cutover hint — all deleted (no + settings edit here anymore). +- **`test_storage_connection`** loses the S3-fields DTO; becomes a + round-trip probe against a named entry: `POST .../test?storage=`. +- **`verify_migration`** deleted; the "Verify integrity" button on the + storage tab is rewired to trigger `blobs_consistency?storage=` + against the migration target (or dropped in favour of the standard + admin/jobs Run button — TBD in slice 6). + +## Slice breakdown + +Single PR is too big; splitting into a coherent sequence. Slices 1-2 are +foundational; the rest layer on top independently within reason. + +| # | Slice | Depends on | Rough size | +|---|---|---|---| +| 1 | Config parser: `OXICLOUD_STORAGE_ENTRIES` + per-entry vars + `_ENCRYPTION_KEY` → `Vec` on `AppConfig`. Legacy synthesis for empty `_ENTRIES` (default entry from legacy flat vars). | — | 1 day | +| 2 | Boot: read `active_backend_name` from `admin_settings`; look up entry; build `blob_backend` via a shared `build_entry_backend(&NamedStorageEntry)` factory (wraps encryption decorator when key present). Fail-fast on missing entry with actionable message. | 1 | ~half day | +| 3 | Migration handler rewrite: params carry `target_name` only. Handler resolves target by name from `storage_settings.build_entry_backend(name)`. `is_source_target_identical` guard refactored to compare NAMES first; keeps physical-identity check as second-line refusal (with encryption-differs-specific message). Retire the migration DTO S3-field body. | 1, 2 | ~half day | +| 4 | Global `migration_readonly` flag on `AppState`, backed by `admin_settings.storage.migration_readonly`. One clause added to `PgAclEngine::check_inner`. Boot-time clear rule (no in-flight + active matches booted → clear). | 2 | ~half day | +| 5 | Cutover state machine: on migration `Completed`, write `active_backend_name = target_name`, keep read-only on. Boot on new backend after operator restart. | 4 | ~half day | +| 6 | Admin storage tab rewrite: list entries, show active, migrate dropdown, read-only banner. Delete Save form + S3 field editors + .env cutover hint. | 1, 3, 4 | 1 day | +| 7 | `?storage=` on `blobs_consistency` + `backend_consistency`. `JobRunArgs.storage` plumbing, `TriggerJobQuery.storage`, entry-resolver at run start, params records probed name. Retire `verify_migration` + its DTO + its route + its handler. | 1, 3 | 1 day | +| 8 | `oxicloud --select-storage ` bare-flag repair command on the main binary. Parses `.env`, verifies entry exists, UPDATEs DB, exits. Boot-time missing-entry error message points at it. See §Fallback. | 2 | ~quarter day | + +**Total: ~5-6 days end to end.** Slices 6 and 7 can proceed in parallel with +each other once 1-5 land. Slice 8 is an ops nicety, could ship whenever. + +## Verification + +Per slice, plus these end-to-end scenarios in Hurl: + +1. **Fresh install, no `_ENTRIES`**: boot uses synthesized `default` entry from + legacy vars, `active_backend_name` unset. `GET /admin/settings/storage` + returns one entry, active. No cutover UI. +2. **Two entries, no active set**: boot picks first in `_ENTRIES`, logs it, + proceeds. Admin panel shows both entries with the picked one marked active. +3. **Change active without migration** (rarely useful but must be safe): + `POST /admin/settings/storage/active-name` (or however the pointer is + exposed) updates the row; next restart boots on the new entry. Existing + blobs on the new entry NOT verified — operator's problem, but + `blobs_consistency` catches the drift on next run. +4. **Migration happy path**: two entries, entry A active, migrate to B. + Read-only comes on. Copy runs. `active_backend_name` flips to B on + completion. Read-only stays on until restart. After restart, live is B, + read-only cleared. +5. **Restart mid-migration**: kill server after checkpoint N. Boot: row Paused, + `active` still A, `migration_readonly` still true. Admin retriggers → + resumes from checkpoint N against target B (resolved by name from params). + Complete → pointer flips → restart → live is B. +6. **In-place encryption rotation refused**: two entries, same S3 bucket, + different encryption keys. Trigger migration → refuses with the specific + error message pointing at the encryption case and the two-step workaround. +7. **`?storage=` on blobs_consistency**: run against `s3_prod` before + cutover. Full walk, `probed_storage` in run row. Then cutover, then rerun + against `local_main` — verifies old backend still has everything. +8. **Unknown storage name**: `POST /admin/jobs/blobs_consistency/trigger?storage=nope` + → 400 with known-names list. No run row created. +9. **Missing entry at boot**: `active_backend_name = "gone"` but `_ENTRIES` + doesn't include it → boot aborts with the specific message pointing at + `oxicloud --select-storage ` (with the available names filled in). + Re-run the binary with `--select-storage local_main` → verifies + updates + DB + exits 0. Restart the server → boots cleanly on `local_main`. +10. **Encryption key invalid**: `OXICLOUD_STORAGE__ENCRYPTION_KEY=badbase64` + → boot aborts with entry name + reason (not valid base64 / wrong length). +11. **Legacy vars alongside `_ENTRIES`**: set `_ENTRIES=foo` AND leave a + stale `OXICLOUD_S3_BUCKET=...` in `.env`. Boot aborts with the full + list of legacy vars detected, tells the admin to remove them (or move + them into `OXICLOUD_STORAGE__S3_BUCKET` form). Removing the + legacy var → next boot succeeds. +12. **Legacy synthesis path**: `_ENTRIES` unset, `OXICLOUD_STORAGE_BACKEND=s3` + + `OXICLOUD_S3_BUCKET=...` set. Boot synthesizes one entry named + `default` from those vars. `storage_entries.len() == 1`, name is + `default`, backend is S3, config carries the flat-var values. + +## Out of scope + +- **Dual-write decorator / zero-downtime migration**: read-only is v1; + dual-write is filed under "if operator demand appears". Would rebuild + `MigrationBlobBackend` (deleted in the recoverable-migration PR — retained + in git for the same reason). +- **In-place encryption key rotation** (same-bucket, different key): refused + by the identity guard; workaround is two-step via temp bucket. Proper fix + (per-generation object naming) is documented inline in the Encryption + section above; deferred until real demand appears. +- **Runtime backend hot-swap without restart**: not attempted. Requires + `Arc>>` indirection at every call site + plus per-request coordination; huge blast radius. Read-only + restart is + the honest answer. +- **Per-user or per-drive storage backends**: everything in this plan is + server-scoped. If per-drive storage becomes a real need, the named-entry + registry is the right substrate but `blob_backend` on AppState becomes a + `EntryResolver` and every call site changes. Not now. +- **DB storage config UI**: retired. Admin panel is a controller (list + + test + migrate + audit), never a persister. If persisted per-entry knobs + become a need (retention days per entry, quota per entry, …), those live + in DB rows keyed by entry name — a small extension, not a return to the + old model. +- **Secret encryption in DB**: `admin_settings.storage.*` secret rows are + deleted along with `save_storage_settings`. If the migration params + approach ever grows to store secrets (e.g., a future "supply the target + creds inline for one-off migrations"), those get encrypted at rest with + a KMS-provided key. Not needed for this plan — params holds only names. + +## Related memory notes + +- `feedback_no_abbreviated_env_vars` — full-word env var names + (`OXICLOUD_STORAGE_local_main_S3_ENDPOINT_URL`, not + `OXICLOUD_STORAGE_local_main_S3_EP`). +- `feedback_config_file_overrides_shell` — `dotenvy` behaviour on explicit + config path. Matters when `_ENTRIES` values are loaded from `--config` vs + shell. +- `project_admin_middleware_layer` — admin ops bypass read-only via existing + middleware; the AuthZ short-circuit added in slice 4 doesn't need any + new admin carve-out. +- `bug_drive_rename_editor_can_do_it` — reminder that + `PgAclEngine::check_inner` is where write-permission short-circuits live; + the new global read-only clause lands next to the per-drive one. +- `docs/plan/job-registry.md` Part 2 — recoverable-run engine that + `storage_migration` runs on; `params` field, resume semantics, boot + sweep. diff --git a/example.env b/example.env index 9c673fe0..f84b531a 100644 --- a/example.env +++ b/example.env @@ -353,8 +353,73 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud #OXICLOUD_PLUGIN_LOG_QUEUE_CAPACITY=1024 # ----------------------------------------------------------------------------- -# STORAGE BACKEND +# STORAGE ENTRIES (multi-entry, recommended) # ----------------------------------------------------------------------------- +# +# Declare one or more NAMED storage backends. The one the app runs on is +# picked from the DB (`admin_settings.storage.active_backend_name`) — the +# admin panel's storage tab flips it, and cross-backend migration is a +# recoverable-run job that copies blobs between two entries. See +# `docs/plan/storage-multi-entry.md` for the full model. +# +# Rules: +# * `OXICLOUD_STORAGE_ENTRIES` is a comma-separated allowlist of names. +# Names must match `[a-z0-9_-]{1,32}` and be unique. Order is +# preserved (the first entry is the fallback when no active pointer +# is set in the DB yet — e.g. fresh install). +# * For each name `N`, the parser reads +# `OXICLOUD_STORAGE__BACKEND` (local | s3 | azure) plus the +# backend-specific fields below. A missing required field aborts +# boot with the exact var name in the error message. +# * Presence of `OXICLOUD_STORAGE__ENCRYPTION_KEY` implies AES-256 +# encryption is enabled on that entry (no separate enable flag). +# Bad base64 / wrong length aborts boot with the entry name. +# * SETTING `_ENTRIES` alongside the legacy flat vars below (e.g. +# `OXICLOUD_STORAGE_BACKEND` + `OXICLOUD_S3_BUCKET`) is a FAIL-FAST +# boot error — pick one mode. Migrate any leftover flat vars into +# per-entry `_STORAGE__*` form. +# +# Example: local disk today, S3 target for a planned migration. +# +#OXICLOUD_STORAGE_ENTRIES=local_main,s3_prod +# +#OXICLOUD_STORAGE_local_main_BACKEND=local +#OXICLOUD_STORAGE_local_main_ROOT_DIR=/srv/oxicloud +# +#OXICLOUD_STORAGE_s3_prod_BACKEND=s3 +#OXICLOUD_STORAGE_s3_prod_S3_BUCKET=my-oxicloud-bucket +#OXICLOUD_STORAGE_s3_prod_S3_REGION=us-east-1 +#OXICLOUD_STORAGE_s3_prod_S3_ENDPOINT_URL=https://s3.example.com +#OXICLOUD_STORAGE_s3_prod_S3_ACCESS_KEY= +#OXICLOUD_STORAGE_s3_prod_S3_SECRET_KEY= +#OXICLOUD_STORAGE_s3_prod_S3_FORCE_PATH_STYLE=false +#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_KEY= # generate: openssl rand -base64 32 +# Cipher declaration — future-proofing. Today only `aes-256-gcm` is +# accepted (and it's the default when `_ENCRYPTION_KEY` is set), so +# this line can be omitted. Explicit here as documentation. +#OXICLOUD_STORAGE_s3_prod_ENCRYPTION_CIPHER=aes-256-gcm +# +# Repair flag: if you rename an entry in .env while the DB still points +# at the old name, boot aborts with an actionable error pointing at: +# +# oxicloud --select-storage +# +# which verifies the entry exists in `_ENTRIES` and updates the DB +# pointer without booting the server. See §Fallback in the plan doc. + +# ----------------------------------------------------------------------------- +# STORAGE BACKEND — DEPRECATED (single-backend flat vars) +# ----------------------------------------------------------------------------- +# +# ⚠️ DEPRECATED. Use the STORAGE ENTRIES section above for new deployments. +# These flat variables still work when `OXICLOUD_STORAGE_ENTRIES` is UNSET — +# the parser then synthesises a single entry named `default` from them AND +# emits a boot-time deprecation warning +# (`storage.legacy_flat_vars_deprecated`) so operators see it in logs. +# Removal target: not yet fixed. Migrate at your convenience by moving +# each `OXICLOUD_STORAGE_BACKEND` / `OXICLOUD_S3_*` / `OXICLOUD_AZURE_*` / +# `OXICLOUD_STORAGE_ENCRYPTION_*` into `OXICLOUD_STORAGE__*` under +# an entry declared in `OXICLOUD_STORAGE_ENTRIES`. # Blob storage backend: local (default), s3, or azure #OXICLOUD_STORAGE_BACKEND=local @@ -400,9 +465,15 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud # Cache directory (default: {STORAGE_PATH}/.blob-cache) #OXICLOUD_STORAGE_CACHE_PATH= -# --- Client-Side Encryption --- +# --- Client-Side Encryption --- DEPRECATED (per-entry key is the new home) # AES-256-GCM encryption applied to blobs before writing to any backend. # WARNING: losing the key means losing all data. Back it up securely. +# +# ⚠️ DEPRECATED. Prefer per-entry `OXICLOUD_STORAGE__ENCRYPTION_KEY` +# under an entry declared in `OXICLOUD_STORAGE_ENTRIES` (see the top +# multi-entry section). The flat vars below still work in +# zero-entries mode and get folded into the synthesised `default` +# entry, alongside a deprecation warning at boot. # Enable at-rest blob encryption (default: false) #OXICLOUD_STORAGE_ENCRYPTION_ENABLED=false diff --git a/examples/bench_favorites_authz.rs b/examples/bench_favorites_authz.rs index c97cce8f..335bb1bc 100644 --- a/examples/bench_favorites_authz.rs +++ b/examples/bench_favorites_authz.rs @@ -186,6 +186,7 @@ fn fresh_engine(pool: &Arc) -> Arc { folder_repo, file_repo, group_repo, + Arc::new(std::sync::atomic::AtomicBool::new(false)), )) } diff --git a/examples/bench_range_seek_authz.rs b/examples/bench_range_seek_authz.rs index 50d5b700..82dfd2eb 100644 --- a/examples/bench_range_seek_authz.rs +++ b/examples/bench_range_seek_authz.rs @@ -176,6 +176,7 @@ fn fresh_engine(pool: &Arc) -> Arc { folder_repo, file_repo, group_repo, + Arc::new(std::sync::atomic::AtomicBool::new(false)), )) } diff --git a/examples/bench_round12_queries.rs b/examples/bench_round12_queries.rs index 3f3d8b53..cb7f25c5 100644 --- a/examples/bench_round12_queries.rs +++ b/examples/bench_round12_queries.rs @@ -769,6 +769,7 @@ fn wopi_engine(pool: &Arc) -> (Arc, Arc) -> (Arc, Arc) -> Arc { folder_repo, file_repo, group_repo, + Arc::new(std::sync::atomic::AtomicBool::new(false)), )) } diff --git a/frontend/src/lib/api/client.ts b/frontend/src/lib/api/client.ts index f388550c..d9024c67 100644 --- a/frontend/src/lib/api/client.ts +++ b/frontend/src/lib/api/client.ts @@ -18,6 +18,15 @@ */ import { getCsrfHeaders } from './csrf'; +import { updateFromHeader } from '$lib/stores/serverStatus.svelte'; + +/** + * Name of the response header the server stamps while a + * maintenance event is live. Case-insensitive on the wire — the + * Fetch API's `Headers.get` matches irrespective of case, so this + * constant matches whatever axum emits. + */ +const SERVER_STATUS_HEADER = 'x-server-status'; const REFRESH_ENDPOINT = '/api/auth/refresh'; @@ -93,6 +102,18 @@ export function createApiFetch(deps: ApiClientDeps): FetchFn { const apiFetch: FetchFn = async (input, init) => { const origin = deps.origin ?? globalThis.location?.origin ?? 'http://localhost'; const response = await rawFetch(input, init); + // Server-status header piggyback — the server stamps + // `x-server-status` on every response while a maintenance + // event is in progress (see middleware::server_status). Read + // it and update the reactive store; the AppShell banner + // subscribes and shows/hides itself. Absent header = nothing + // happening; the update fn resets the store to default in + // that case so a lingering banner disappears. + // + // Runs on EVERY response including a 401 (below) so a session + // refresh doesn't accidentally clear a live banner. + updateFromHeader(response.headers.get(SERVER_STATUS_HEADER)); + if (response.status !== 401) return response; const urlStr = urlString(input as RequestInfo | URL); @@ -104,7 +125,9 @@ export function createApiFetch(deps: ApiClientDeps): FetchFn { onSessionExpired(); throw new Error('Session expired'); } - return rawFetch(input, init); + const retryResponse = await rawFetch(input, init); + updateFromHeader(retryResponse.headers.get(SERVER_STATUS_HEADER)); + return retryResponse; }; return apiFetch; diff --git a/frontend/src/lib/api/endpoints/admin.test.ts b/frontend/src/lib/api/endpoints/admin.test.ts index 1df88227..5c4a6977 100644 --- a/frontend/src/lib/api/endpoints/admin.test.ts +++ b/frontend/src/lib/api/endpoints/admin.test.ts @@ -104,15 +104,8 @@ describe('admin test/probe endpoints', () => { await expect(admin.testStorage({ backend: 's3' })).resolves.toMatchObject({ connected: true }); }); - it('verifyMigration fills defaults and throws on error', async () => { - fetchMock.mockResolvedValue(okRes({ passed: true })); - await expect(admin.verifyMigration(10)).resolves.toMatchObject({ - passed: true, - sample_checked: 0 - }); - fetchMock.mockResolvedValue(errRes(500, {})); - await expect(admin.verifyMigration()).rejects.toThrow(/verify failed/); - }); + // verifyMigration retired in slice 7 of docs/plan/storage-multi-entry.md. + // Superseded by `POST /api/admin/jobs/blobs_consistency/trigger?storage=`. it('installPlugin posts a FormData bundle', async () => { fetchMock.mockResolvedValue(okRes({ id: 'com.example.hello' })); diff --git a/frontend/src/lib/api/endpoints/admin.ts b/frontend/src/lib/api/endpoints/admin.ts index 8d9ca09b..a4001438 100644 --- a/frontend/src/lib/api/endpoints/admin.ts +++ b/frontend/src/lib/api/endpoints/admin.ts @@ -384,13 +384,28 @@ export interface SmtpTestResult { error?: string; } -/** Result of POST .../settings/storage/test — the S3 connection probe. */ +/** + * Result of POST .../settings/storage/test. Combines reachability + * (`connected` — HEAD bucket / statfs) with a full read/write round- + * trip (`roundtrip_passed` — PUT + GET + verify + DELETE). Overall + * pass = both true. Round-trip fields are absent when the round-trip + * wasn't attempted (typically because reachability already failed). + * `phase_reached` names the last successful round-trip step: + * `initialize` | `put_ok` | `exists_ok` | `get_ok` | `verify_ok` | + * `cleanup_ok`. + */ export interface StorageTestResult { connected?: boolean; success?: boolean; backend_type?: string; available_bytes?: number | null; message?: string; + roundtrip_passed?: boolean; + phase_reached?: string; + bytes_written?: number; + bytes_read?: number; + roundtrip_elapsed_ms?: number; + cleanup_ok?: boolean; } export async function sendSmtpTest(to: string): Promise { @@ -451,19 +466,26 @@ export function saveOidc(body: Record): Promise { // ── Storage settings + migration ─────────────────────────────────────────── -export interface StorageSettings { +export interface StorageEntrySummary { + name: string; backend: string; - s3_endpoint_url?: string | null; - s3_bucket?: string | null; - s3_region?: string | null; - s3_access_key_set?: boolean; - s3_secret_key_set?: boolean; - s3_force_path_style?: boolean; - env_overrides?: string[]; + is_active: boolean; + encryption_enabled: boolean; + /** Human-readable physical hint (root_dir / bucket / container). */ + location_hint?: string | null; +} + +export interface StorageSettings { + // Live stats — what the running process reports. current_backend?: string; total_blobs?: number; total_bytes_stored?: number; dedup_ratio?: number; + // Multi-entry view (slice 6 of docs/plan/storage-multi-entry.md). + // `entries` is empty for the legacy zero-entries path. + entries?: StorageEntrySummary[]; + active_entry_name?: string; + migration_readonly?: boolean; } export function getStorageSettings(): Promise { @@ -497,45 +519,33 @@ export function getMigration(): Promise { return apiJson('/api/admin/storage/migration', { credentials: 'same-origin' }); } -export function migrationAction(action: 'start' | 'pause' | 'resume' | 'complete'): Promise { - const body = action === 'start' ? { concurrency: 4 } : {}; +export function migrationAction( + action: 'start' | 'pause' | 'resume', + targetName?: string +): Promise { + // `complete` was retired when the migration became a recoverable + // job — Completed is the terminal `RunSummary.status`; there's + // nothing left to acknowledge. Post-migration cutover happens on + // operator restart (server re-boots on active_backend_name = the + // new entry; the boot-clear rule drops migration_readonly). + // + // `start` REQUIRES `targetName` in multi-entry mode — the backend + // rejects an unnamed start with 400 (see StartMigrationDto). + // `pause` and `resume` take no body (resume reads target_name + // from the paused run's params). + const body: Record = + action === 'start' ? { target_name: targetName ?? '', concurrency: 4 } : {}; return mutate(`/api/admin/storage/migration/${action}`, 'POST', body); } -/** Result of a `verify` integrity check (POST .../migration/verify). */ -export interface MigrationVerifyResult { - passed: boolean; - sample_checked: number; - pg_blob_count: number; - missing_in_target: string[]; - size_mismatches: string[]; -} - -/** - * Run an integrity verification pass over a sample of migrated blobs. Unlike - * the other migration actions this returns a structured result that the caller - * renders (passed / sample-checked / missing / size-mismatch counts). - */ -export async function verifyMigration(sampleSize = 100): Promise { - const res = await apiFetch('/api/admin/storage/migration/verify', { - method: 'POST', - credentials: 'same-origin', - headers: { ...JSON_HEADERS, ...getCsrfHeaders() }, - body: JSON.stringify({ sample_size: sampleSize }) - }); - if (!res.ok) { - const e = (await res.json().catch(() => ({}))) as { message?: string }; - throw new Error(e.message || `verify failed: ${res.status}`); - } - const r = (await res.json()) as Partial; - return { - passed: r.passed ?? false, - sample_checked: r.sample_checked ?? 0, - pg_blob_count: r.pg_blob_count ?? 0, - missing_in_target: r.missing_in_target ?? [], - size_mismatches: r.size_mismatches ?? [] - }; -} +// verifyMigration + MigrationVerifyResult retired in slice 7 of +// docs/plan/storage-multi-entry.md — the corresponding backend +// endpoint's sample-based check is superseded by +// `POST /api/admin/jobs/blobs_consistency/trigger?storage=`, +// which does a full walk against any named entry and integrates +// with the standard runs / findings admin surface. Trigger from +// the Jobs tab; the Storage tab drops the "Verify integrity" +// button. // ── Plugins ───────────────────────────────────────────────────────────── diff --git a/frontend/src/lib/api/endpoints/adminJobs.ts b/frontend/src/lib/api/endpoints/adminJobs.ts index fb7f4764..97db6419 100644 --- a/frontend/src/lib/api/endpoints/adminJobs.ts +++ b/frontend/src/lib/api/endpoints/adminJobs.ts @@ -53,11 +53,16 @@ export function listJobs(): Promise { */ export async function triggerJob( name: string, - opts: { force?: boolean; deep?: boolean } = {} + opts: { force?: boolean; deep?: boolean; storage?: string } = {} ): Promise { const params = new URLSearchParams(); if (opts.force) params.set('force', 'true'); if (opts.deep) params.set('deep', 'true'); + // `storage` scopes tenants that respect JobRunArgs.storage — + // currently blobs_consistency / backend_consistency (probes the + // named entry instead of the live backend). See + // `docs/plan/storage-multi-entry.md` slice 7. + if (opts.storage) params.set('storage', opts.storage); const q = params.toString(); const url = `/api/admin/jobs/${encodeURIComponent(name)}/trigger${q ? `?${q}` : ''}`; const res = await apiFetch(url, { diff --git a/frontend/src/lib/components/AppShell.svelte b/frontend/src/lib/components/AppShell.svelte index 934610e8..d669b4dd 100644 --- a/frontend/src/lib/components/AppShell.svelte +++ b/frontend/src/lib/components/AppShell.svelte @@ -11,10 +11,12 @@ import type { FileItem, FolderItem, ItemType } from '$lib/api/types'; import { lazyComponent } from '$lib/composables/lazyComponent.svelte'; import DrivePicker from '$lib/components/DrivePicker.svelte'; + import ReadOnlyBanner from '$lib/components/ReadOnlyBanner.svelte'; import Icon from '$lib/icons/Icon.svelte'; import { dateTimeFormatFor, iconNameFromClass } from '$lib/utils/display'; import { userInitials, avatarColorIndex } from '$lib/utils/avatar'; import { i18n, LANGUAGES, setLocale, t, type Locale } from '$lib/i18n/index.svelte'; + import { serverStatus } from '$lib/stores/serverStatus.svelte'; import { apiFetch } from '$lib/api/client'; import { dialogs } from '$lib/stores/dialogs.svelte'; import { files as filesStore } from '$lib/stores/files.svelte'; @@ -1025,6 +1027,21 @@
+ + {#if serverStatus().readonly} + + {/if} {@render children()}
diff --git a/frontend/src/lib/components/ReadOnlyBanner.svelte b/frontend/src/lib/components/ReadOnlyBanner.svelte index cfee3dae..d7fd5491 100644 --- a/frontend/src/lib/components/ReadOnlyBanner.svelte +++ b/frontend/src/lib/components/ReadOnlyBanner.svelte @@ -1,6 +1,8 @@
- {#if driveName} + {#if variant === 'maintenance'} + {t('server_status.readonly_title', 'Server maintenance in progress')} + {:else if driveName} {t( 'drive.read_only_banner.title_named', { name: driveName }, @@ -56,10 +85,30 @@ {/if} - {t( - 'drive.read_only_banner.body', - 'Uploads, edits, deletes, renames, sharing and membership changes are refused. Reads and downloads keep working. Contact an administrator to un-freeze the drive.' - )} + {#if variant === 'maintenance'} + {#if progress} + {t( + 'server_status.readonly_progress', + { + target: progress.target, + migrated: progress.migrated, + total: progress.total, + percent: progress.percent + }, + 'Migrating storage to `{{target}}` — {{percent}}% ({{migrated}} / {{total}} blobs). Uploads, renames, deletes, and shares are refused; reads and downloads work as normal.' + )} + {:else} + {t( + 'server_status.readonly_body', + 'Uploads, renames, deletes, and shares are refused temporarily. Reads and downloads work as normal.' + )} + {/if} + {:else} + {t( + 'drive.read_only_banner.body', + 'Uploads, edits, deletes, renames, sharing and membership changes are refused. Reads and downloads keep working. Contact an administrator to un-freeze the drive.' + )} + {/if}
diff --git a/frontend/src/lib/stores/serverStatus.svelte.ts b/frontend/src/lib/stores/serverStatus.svelte.ts new file mode 100644 index 00000000..a16d449b --- /dev/null +++ b/frontend/src/lib/stores/serverStatus.svelte.ts @@ -0,0 +1,67 @@ +/** + * Reactive server-status store. + * + * Populated by the `apiFetch` wrapper, which reads the + * `x-server-status` header off every API response and calls + * `updateFromHeader(...)`. When no migration is running the header + * is absent and the store stays at its default (readonly=false, no + * migration info). See `middleware::server_status` on the server + * for the header spec. + * + * The AppShell subscribes to this store to show/hide the + * maintenance banner without polling — the state travels back to + * the client on the piggyback of whatever API request the user was + * making anyway. Zero extra network cost. + */ + +/** + * JSON shape emitted in the `x-server-status` header. Optional + * `migration` field is present only while a migration is running. + */ +export interface ServerStatus { + readonly: boolean; + migration?: { + target: string; + migrated: number; + total: number; + percent: number; + }; +} + +const DEFAULT: ServerStatus = { readonly: false }; + +// Rune-based reactive state — `$state` in a `.svelte.ts` module. +let current = $state(DEFAULT); + +/** Current server status. Reactively updates when apiFetch sees a new header. */ +export function serverStatus(): ServerStatus { + return current; +} + +/** + * Parse the raw header value and update the store. Silently + * tolerates a missing header (resets to default: nothing to + * broadcast means nothing wrong) and a malformed one (keeps the + * previous value rather than surface a parse error to users). + * + * Called by `apiFetch` after every response — see `client.ts`. + */ +export function updateFromHeader(rawHeader: string | null): void { + if (rawHeader == null) { + // No header on this response = server not in maintenance + // mode = reset the store to the default so any lingering + // banner disappears. Cheap idempotent write. + if (current.readonly || current.migration) current = DEFAULT; + return; + } + try { + const parsed = JSON.parse(rawHeader) as ServerStatus; + // Basic shape validation — server should never send a + // missing `readonly`, but be defensive. + if (typeof parsed.readonly === 'boolean') { + current = parsed; + } + } catch { + // Malformed header — keep previous state rather than churn. + } +} diff --git a/frontend/src/routes/admin/[[tab]]/+page.svelte b/frontend/src/routes/admin/[[tab]]/+page.svelte index a8bb88ea..0e617957 100644 --- a/frontend/src/routes/admin/[[tab]]/+page.svelte +++ b/frontend/src/routes/admin/[[tab]]/+page.svelte @@ -26,7 +26,6 @@ resetUserPassword, saveOidc, savePluginRetention, - saveStorage, sendSmtpTest, setPluginEnabled, setRegistrationEnabled, @@ -35,7 +34,6 @@ setUserRole, testOidc, testStorage, - verifyMigration, createExternalMount, deleteExternalMount, listExternalMounts, @@ -44,7 +42,6 @@ type AdminDashboard, type GeneratedKey, type MigrationStatus, - type MigrationVerifyResult, type OidcSettings, type OidcTestResult, type PluginInfo, @@ -77,6 +74,7 @@ DrivePoliciesPartial, User } from '$lib/api/types'; + import { triggerJob } from '$lib/api/endpoints/adminJobs'; import AdminJobsPanel from '$lib/components/AdminJobsPanel.svelte'; import Icon from '$lib/icons/Icon.svelte'; import Modal from '$lib/components/Modal.svelte'; @@ -372,118 +370,72 @@ } } - // Storage - const STORAGE_PRESETS: Record = - { - custom: { endpoint: '', region: '', pathStyle: false }, - aws: { endpoint: '', region: 'us-east-1', pathStyle: false }, - backblaze: { - endpoint: 'https://s3.{region}.backblazeb2.com', - region: 'us-west-004', - pathStyle: false - }, - 'cloudflare-r2': { - endpoint: 'https://{accountId}.r2.cloudflarestorage.com', - region: 'auto', - pathStyle: true - }, - minio: { endpoint: 'http://localhost:9000', region: 'us-east-1', pathStyle: true }, - digitalocean: { - endpoint: 'https://{region}.digitaloceanspaces.com', - region: 'nyc3', - pathStyle: false - }, - wasabi: { - endpoint: 'https://s3.{region}.wasabisys.com', - region: 'us-east-1', - pathStyle: false - } - }; + // Storage — multi-entry read-only view. + // + // Post `docs/plan/storage-multi-entry.md`, the .env is the SOLE + // place to declare backends. The admin storage tab is now: + // - a read-only list of the entries the server booted with, + // - a per-entry test button (round-trip against that entry), + // - a per-entry audit action (triggers blobs_consistency?storage=), + // - a per-non-active migrate+activate button, + // - the migration status line + cutover hint. + // No form. No save. The retired save endpoint / DTO are still on + // the backend during the deprecation window but the UI never + // hits them. let storage = $state(null); - let sForm = $state({ - backend: 'local', - preset: 'custom', - endpoint: '', - bucket: '', - region: '', - accessKey: '', - secretKey: '', - pathStyle: false - }); let storageMsg = $state<{ text: string; ok: boolean } | null>(null); - let storageBusy = $state(false); + // Per-entry test state — keyed by entry name so the buttons don't + // step on each other and the last result stays visible per row. + let entryTest = $state< + Record + >({}); async function loadStorage() { try { storage = await getStorageSettings(); - sForm = { - backend: storage.backend ?? 'local', - preset: 'custom', - endpoint: storage.s3_endpoint_url ?? '', - bucket: storage.s3_bucket ?? '', - region: storage.s3_region ?? '', - accessKey: '', - secretKey: '', - pathStyle: storage.s3_force_path_style ?? false + } catch (e) { + storageMsg = { text: errorMessage(e), ok: false }; + } + } + + async function doTestEntry(name: string) { + entryTest = { ...entryTest, [name]: { busy: true } }; + try { + const r: StorageTestResult = await testStorage({ entry_name: name }); + entryTest = { ...entryTest, [name]: { busy: false, result: r } }; + } catch (e) { + entryTest = { ...entryTest, [name]: { busy: false, error: errorMessage(e) } }; + } + } + + async function doAuditEntry(name: string) { + try { + await triggerJob('blobs_consistency', { storage: name }); + storageMsg = { + text: t( + 'admin.storage_audit_triggered', + { name }, + 'blobs_consistency triggered for `{{name}}` — watch it on the Jobs tab.' + ), + ok: true }; } catch (e) { storageMsg = { text: errorMessage(e), ok: false }; } } - function applyPreset() { - const p = STORAGE_PRESETS[sForm.preset]; - if (!p) return; - if (p.endpoint) sForm.endpoint = p.endpoint; - if (p.region) sForm.region = p.region; - sForm.pathStyle = p.pathStyle; - } - function storageBody() { - return { - backend: sForm.backend, - s3_endpoint_url: sForm.endpoint.trim() || null, - s3_bucket: sForm.bucket.trim() || null, - s3_region: sForm.region.trim() || null, - s3_access_key: sForm.accessKey || null, - s3_secret_key: sForm.secretKey || null, - s3_force_path_style: sForm.pathStyle - }; - } - async function doSaveStorage() { - storageBusy = true; - storageMsg = null; - try { - await saveStorage(storageBody()); - storageMsg = { text: t('admin.storage_saved', 'Storage settings saved.'), ok: true }; - await loadStorage(); - } catch (e) { - storageMsg = { text: errorMessage(e), ok: false }; - } finally { - storageBusy = false; - } - } - async function doTestStorage() { - storageBusy = true; - storageMsg = null; - try { - const r: StorageTestResult = await testStorage(storageBody()); - const ok = r.connected ?? r.success ?? false; - if (ok) { - let text = t('admin.storage_test_success', 'Connection successful'); - if (r.backend_type) text += ` (${r.backend_type})`; - if (r.available_bytes != null) - text += ` — ${formatBytes(r.available_bytes)} ${t('admin.available', 'available')}`; - storageMsg = { text, ok: true }; - } else { - storageMsg = { - text: `${t('admin.storage_test_failure', 'Connection failed')}: ${r.message ?? ''}`, - ok: false - }; - } - } catch (e) { - storageMsg = { text: errorMessage(e), ok: false }; - } finally { - storageBusy = false; - } + + async function doMigrateActivate(name: string) { + if ( + !confirm( + t( + 'admin.storage_migrate_confirm', + { name }, + 'Migrate all blobs to `{{name}}` and set it as the active entry? The server enters read-only mode during the copy; the live backend swaps automatically on completion (no restart needed).' + ) + ) + ) + return; + await doMigration('start', name); } // Migration @@ -496,43 +448,94 @@ migrationTimer = null; } } + // Polling model for the migration flow: + // 1. `lastMigrationStatus` — last observed status. Storage + // state is refreshed on any transition so the readonly + // banner + active-entry indicators track the server without + // the admin having to reload the tab. + // 2. `sawActive` — flips true the first time we observe + // `running` or `paused` after a user action. We only STOP + // polling on `idle` / `completed` / `failed` AFTER + // `sawActive` is true — otherwise a trigger endpoint's + // instant-202 response (the run row hasn't landed in the + // DB yet) would kill the poll loop before the migration + // even started, and the banner + active-entry update + // would never show up until the admin manually refreshed. + // 3. `pendingSince` — timestamp of the last user action. + // Bounds how long we keep polling on `idle` while waiting + // for the run row to appear. If the run never opens within + // the grace window (60 s — dispatch spawn + DB insert + // normally takes < 100 ms), we give up. + let lastMigrationStatus: string | undefined; + let sawActive = false; + let pendingSince: number | undefined; + const PENDING_GRACE_MS = 60_000; async function loadMigration() { try { migration = await getMigration(); - if (migration.status === 'running') { + const status = migration.status; + const active = status === 'running' || status === 'paused'; + if (active) sawActive = true; + + // Refresh storage on any status change so the readonly + // banner + active-entry indicators reflect current + // server state. + if (status !== lastMigrationStatus) { + lastMigrationStatus = status; + void loadStorage(); + } + + // Keep polling while the migration is active OR while + // we're within the grace window waiting for a + // user-triggered run to appear. + const withinGrace = + !sawActive && pendingSince != null && performance.now() - pendingSince < PENDING_GRACE_MS; + if (active || withinGrace) { if (!migrationTimer) migrationTimer = setInterval(loadMigration, 5000); } else { + // Not active and (either we've already seen it run OR + // the grace window ran out) → stop polling. stopMigrationPoll(); + pendingSince = undefined; } } catch { stopMigrationPoll(); + pendingSince = undefined; } } - async function doMigration(action: 'start' | 'pause' | 'resume' | 'complete') { + async function doMigration(action: 'start' | 'pause' | 'resume', targetName?: string) { try { - await migrationAction(action); + await migrationAction(action, targetName); + // Reset the status memo so the very next `loadMigration` + // tick unconditionally reloads storage (an admin-triggered + // action is exactly when the readonly flag flips). + lastMigrationStatus = undefined; + sawActive = false; + pendingSince = performance.now(); await loadMigration(); } catch (e) { reportError(e); } } - // Migration integrity verification (separate result panel). - let verifyResult = $state(null); - let verifyError = $state(null); - let verifying = $state(false); - async function doVerify() { - verifying = true; - verifyResult = null; - verifyError = null; - try { - verifyResult = await verifyMigration(100); - } catch (e) { - verifyError = errorMessage(e); - } finally { - verifying = false; - } - } + // The old target-name picker state was retired — the entries + // table now has per-row "Migrate & activate" buttons on + // non-active entries. Simpler mental model; no picker to sync. + + // Retired: the .env cutover-hint state (cutoverPending + + // cutoverEnvLines + cutoverCopied + copyCutoverEnv). It served + // the pre-multi-entry flow that made admins paste env vars + // into .env after migration. Post-multi-entry, the server + // writes `active_backend_name` to the DB automatically on + // migration completion; the operator just restarts. The new + // short "restart to switch" hint is rendered inline in the + // entries card template, no derived state needed. + + // Migration integrity verification retired in slice 7 — the + // sample-based /storage/migration/verify endpoint is replaced by + // `POST /api/admin/jobs/blobs_consistency/trigger?storage=`, + // a full walk. Operators trigger it from the Jobs tab. + const migrationPct = $derived( migration && migration.total_blobs > 0 ? Math.round((migration.migrated_blobs / migration.total_blobs) * 100) @@ -1922,133 +1925,42 @@ {/if} {:else if tab === 'storage'} +
-

{t('admin.storage_tab', 'Storage')}

+

{t('admin.storage_tab', 'Storage entries')}

+

+ {t( + 'admin.storage_move_hint', + 'To move to another backend storage: declare a new entry in your `.env` (keep the current one), restart the server so it picks it up, then trigger a migration from this page. Cutover happens automatically when the copy completes — no second restart needed.' + )} +

{#if !storage}

{t('common.loading', 'Loading…')}

- {:else} -
(e.preventDefault(), doSaveStorage())} - > - - {#if sForm.backend === 's3'} - - - - - - - - {/if} - {#if storageMsg}

- {storageMsg.text} -

{/if} -
- - {#if sForm.backend === 's3'} - - {/if} -
-
+ {:else if !storage.entries || storage.entries.length === 0} + +

+ + {t( + 'admin.storage_no_entries', + { backend: storage.current_backend ?? '?' }, + 'No OXICLOUD_STORAGE_ENTRIES declared. Running on the legacy single-backend fallback ({{backend}}). Migrate to the multi-entry model — see docs/config/env.md.' + )} +

{t('admin.storage_current', 'Current backend')}
{storage.current_backend ?? '—'}
@@ -2061,134 +1973,233 @@
{t('admin.storage_dedup', 'Dedup ratio')}
{storage.dedup_ratio != null ? `${storage.dedup_ratio.toFixed(2)}x` : '—'}
- {/if} -
- -
-

{t('admin.migration', 'Storage migration')}

- {#if !migration} -

{t('common.loading', 'Loading…')}

{:else} -

{t('admin.status', 'Status')}: {migration.status}

- {#if migration.total_blobs > 0} -
-
+ {#if storage.migration_readonly} +
+

+ + {t('admin.mig_readonly_title', 'Server in migration read-only mode')} +

+

+ {t( + 'admin.mig_readonly_body', + 'All writes (upload, rename, delete, share) are refused until the migration completes. Reads (browse, download) are unaffected. When the copy finishes the server switches to the new backend automatically — no restart needed.' + )} +

+ {/if} + + + {@const migrationInFlight = + migration != null && (migration.status === 'running' || migration.status === 'paused')} +
+ {#each storage.entries as entry (entry.name)} + {@const test = entryTest[entry.name]} +
+
+
+ {entry.name} + {#if entry.is_active} + + + {t('admin.entry_active', 'active')} + + {:else} + + {t('admin.entry_inactive', 'available')} + + {/if} + {#if entry.encryption_enabled} + + AES-256 + + {/if} +
+ +
+ + + {#if !entry.is_active && !migrationInFlight} + + {:else} + + {/if} +
+
+
+
{t('admin.entry_backend', 'Backend')}
+
{entry.backend}
+
{t('admin.entry_location', 'Location')}
+
{entry.location_hint ?? '—'}
+ {#if entry.is_active} +
{t('admin.storage_blobs', 'Blobs')}
+
{storage.total_blobs ?? '—'}
+
{t('admin.storage_size', 'Stored')}
+
+ {storage.total_bytes_stored != null + ? formatBytes(storage.total_bytes_stored) + : '—'} +
+
{t('admin.storage_dedup', 'Dedup ratio')}
+
+ {storage.dedup_ratio != null ? `${storage.dedup_ratio.toFixed(2)}x` : '—'} +
+ {/if} +
+ {#if test?.result != null || test?.error != null} +
+ {#if test.error} + {test.error} + {:else if test.result} + {@const ok = test.result.connected ?? false} + {@const rt = test.result.roundtrip_elapsed_ms} + {@const cleanup = test.result.cleanup_ok} + + + {ok + ? t('admin.storage_test_success', 'Read/write OK') + : t('admin.storage_test_failure', 'Test failed')} + {#if rt != null} + · {t('admin.storage_test_elapsed', { ms: rt }, '{{ms}} ms')} + {/if} + {#if cleanup === false} + · ⚠ {t('admin.storage_test_cleanup_warn', 'cleanup DELETE failed')} + {/if} + {#if !ok} + — {test.result.message} + {/if} + + {/if} +
+ {/if} +
+ {/each} +
+ + +

- {migration.migrated_blobs} / {migration.total_blobs} ({migrationPct}%) · - {formatBytes(migration.migrated_bytes)} - {#if migration.throughput_bytes_per_sec && migration.status === 'running'} - · {formatBytes(Math.round(migration.throughput_bytes_per_sec))}/s + {t('admin.mig_status', 'Migration status')}: + {migration?.status ?? '—'} + {#if migration?.status === 'running'} + {/if} - {#if migrationEtaMin != null} - · {t('admin.mig_eta', { min: migrationEtaMin }, `~${migrationEtaMin} min remaining`)} + {#if migration?.status === 'paused'} + {/if}

- {/if} - {#if migration.failed_blobs && migration.failed_blobs.length > 0} -
- - {t( - 'admin.mig_failed', - { n: migration.failed_blobs.length }, - `${migration.failed_blobs.length} failed blobs` - )} - - 
{migration.failed_blobs.join('\n')}
-
- {/if} -
- - {#if migration.status !== 'running' && migration.status !== 'paused' && migration.status !== 'completed'} - + {#if migration && migration.total_blobs > 0} +
+
+
+

+ {migration.migrated_blobs} / {migration.total_blobs} ({migrationPct}%) + {#if migrationEtaMin != null} + · {t( + 'admin.mig_eta', + { min: migrationEtaMin }, + `~${migrationEtaMin} min remaining` + )} + {/if} +

{/if} - {#if migration.status === 'running'} - - {/if} - {#if migration.status === 'paused'} - - {/if} - - {#if migration.status === 'completed'} - - + {#if migration?.failed_blobs && migration.failed_blobs.length > 0} +
+ + {t( + 'admin.mig_failed', + { n: migration.failed_blobs.length }, + `${migration.failed_blobs.length} failed blobs` + )} + + 
{migration.failed_blobs.join('\n')}
+
{/if}
- {#if verifyError} -
- {verifyError} -
- {:else if verifyResult} -
- - - {verifyResult.passed - ? t('admin.mig_verify_passed', 'Verification passed') - : t('admin.mig_verify_failed', 'Verification failed')} - - {#if verifyResult.passed} -

- {t( - 'admin.mig_verify_summary', - { checked: verifyResult.sample_checked, total: verifyResult.pg_blob_count }, - '{{checked}} blobs checked, {{total}} total in database' - )} -

- {:else} -

- {[ - verifyResult.missing_in_target.length - ? t( - 'admin.mig_verify_missing', - { n: verifyResult.missing_in_target.length }, - '{{n}} missing' - ) - : '', - verifyResult.size_mismatches.length - ? t( - 'admin.mig_verify_mismatch', - { n: verifyResult.size_mismatches.length }, - '{{n}} size mismatches' - ) - : '' - ] - .filter(Boolean) - .join(', ')} -

- {/if} -
- {/if} + + {/if} + {#if storageMsg} +

{storageMsg.text}

{/if}
@@ -2197,7 +2208,7 @@

{t( 'admin.encryption_hint', - 'Generate an AES-256 key for at-rest blob encryption, then set it as OXICLOUD_STORAGE_ENCRYPTION_KEY in your server environment.' + 'Generate an AES-256 key for at-rest blob encryption. Set it as OXICLOUD_STORAGE__ENCRYPTION_KEY under an entry declared in OXICLOUD_STORAGE_ENTRIES — presence of the key implies encryption is enabled on that entry (no separate flag).' )}