diff --git a/docs/architecture/backend-storage.md b/docs/architecture/backend-storage.md index a7b1d1b1..544ac22f 100644 --- a/docs/architecture/backend-storage.md +++ b/docs/architecture/backend-storage.md @@ -234,27 +234,38 @@ can be safely dropped. --- -## 4. Blob consistency (`blobs_consistency`) +## 4. Blob consistency — two jobs, split by what they read -Read-only recoverable job that walks `storage.blobs` and reports -divergence between the DB registry and the physical backend. +The registry side and the physical side are separate tenants. They +used to be one, with `blobs_consistency` probing the backend once per +row; that probe found strictly less than the merge-join below, at N +round-trips instead of one enumeration, so it was removed. -### Shallow mode (default) +### `blobs_consistency` — database only -Per row: +Walks `storage.blobs` and compares `ref_count` against the reference +count computed from `storage.files.blob_hash` + +`chunk_manifests.chunk_hashes[]`. On mismatch: `refcount_mismatch` +(severity `inconsistent`), repairable under `?repair=true`. -- `blob_exists(hash)` on the active backend → if false, record - `blob_missing_from_backend` (severity `data_loss`) -- Compare `ref_count` against the actual reference count computed - from `SUM` over `storage.files.blob_hash` + `chunk_manifests.chunk_hashes[]` - → if mismatch, record `refcount_mismatch` (severity `inconsistent`) +It opens no backend and makes no network call. `?storage=` and +`?deep=true` are inert. Cost is one aggregate SQL per row. -Cost: one existence probe + one aggregate SQL per row. Fast on -S3/Azure (single HEAD). +### `backend_consistency` — everything physical -### Deep mode (`?deep=true`) +Merge-joins the backend's enumeration against `storage.blobs`, both +ordered by hash, yielding both deltas in one pass: -Adds a full read of every blob: +- bytes with no registry row → `orphan_blob` (severity `inconsistent`) +- a registry row with no bytes → `blob_missing_from_backend` + (severity `data_loss`) + +`?storage=` scopes it to any declared entry rather than the live +backend. + +### Deep mode (`?deep=true`, on `backend_consistency`) + +For every hash present on both sides, adds a full read: - Stream the blob through `EncryptedBlobBackend::get_blob_stream` (strips header, decrypts if needed, applies BLAKE3 rescue for diff --git a/docs/config/admin-settings.md b/docs/config/admin-settings.md index 589eac3a..e80b6820 100644 --- a/docs/config/admin-settings.md +++ b/docs/config/admin-settings.md @@ -100,12 +100,16 @@ This one-shot repair command re-runs the same env-parse the server does at boot, ### Auditing entries other than the active one -`blobs_consistency` and `backend_consistency` (recoverable jobs on the Jobs tab) accept `?storage=` to probe any declared entry — not just the live one. Use this to verify a migration target before cutover, or to audit an old backend after cutover but before decommissioning: +`backend_consistency` (a recoverable job on the Jobs tab) accepts `?storage=` to audit any declared entry — not just the live one. Use this to verify a migration target before cutover, or to audit an old backend after cutover but before decommissioning: ``` -POST /api/admin/jobs/blobs_consistency/trigger?storage= +POST /api/admin/jobs/backend_consistency/trigger?storage= ``` +Add `?deep=true` to also read every blob back and re-hash it, which catches silent bit-rot. That is a full read of the entry and can take hours. + +`blobs_consistency` does *not* accept `?storage=`: it only reads the database, so there is no entry for it to scope. + Unknown names 400 at the HTTP layer. ## Data Storage diff --git a/docs/config/env.md b/docs/config/env.md index ec158e84..7ddee4f7 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -18,6 +18,7 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Direct (non-chunked) uploads stream straight into the blob store and need no spool directory. Placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_REUSE_PORT` | `false` | Enable `SO_REUSEPORT` so multiple processes can share the same port. **Disabled by default** — a second accidental instance will fail with "address already in use". Enable only for deliberate multi-worker setups (process supervisor, rolling restart). Not supported on Windows. | | `OXICLOUD_METRICS_LISTEN` | (unset) | Prometheus `/metrics` listener address (e.g. `127.0.0.1:9090`, IPv6 allowed as `[::1]:9090`). **Unset = disabled**: no `/metrics` endpoint is bound and no metrics recorder is installed (zero runtime cost). When set, a separate HTTP listener on this address serves the text-format scrape. **Deliberately NOT merged into the main API** — no auth, CSRF, or DPoP layer in front. Bind to loopback or a private interface unless you intend to expose metrics publicly. Starter counters: `oxicloud_dpop_verify_failed_total{reason}`, `oxicloud_dpop_proof_missing_total`, `oxicloud_dpop_header_missing_on_bound_session_total`, `oxicloud_dpop_replay_detected_total`, `oxicloud_dpop_nonce_challenges_issued_total`. | +| `OXICLOUD_STARTUP_JOBS` | `thumb_derived_import?repair=true,thumb_attached_import?repair=true` | Background jobs dispatched once at boot, comma-separated, each `name` or `name?flag=true` using the same syntax as `POST /api/admin/jobs/{name}/trigger`. Flags: `force`, `deep`, `repair`, `storage`. **The default migrates thumbnails out of the legacy `.thumbnails/` directory and deletes the originals**, so the migration completes without anyone triggering it from the admin panel; each sidecar is read back through the normal stack before it is unlinked, and every deletion is audited. An explicit value **replaces** the default; set it empty (`OXICLOUD_STARTUP_JOBS=`) to disable startup jobs, or to `thumb_derived_import,thumb_attached_import` to import without deleting. **Non-blocking** — readiness never waits on a job; entries run sequentially in the background. **Fail-fast** — an unknown job name or flag panics at boot, because a silently-dropped entry means a migration that never runs. A run interrupted by a restart resumes from its cursor on the next boot, so a long migration finishes across restarts. Safe to leave at the default: the jobs are idempotent, and once drained a run does nothing. See [Thumbnail Migration](./thumbnail-migration.md) for the upgrade runbook. | ## Database diff --git a/docs/config/index.md b/docs/config/index.md index 35d424f2..c47a6cf1 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -7,6 +7,7 @@ OxiCloud is configured entirely via **environment variables** (no config files n - [Deployment & Docker](/config/deployment) — Docker Compose, Kubernetes Helm chart, image details - [Environment Variables](/config/env) — complete reference of all `OXICLOUD_*` variables - [Storage Fine Tuning](/config/storage-fine-tuning) — sizing the upload caps + spool directories; tmpfs vs real disk; NVMe split layouts +- [Thumbnail Migration](/config/thumbnail-migration) — upgrading past `.thumbnails/`: what runs on first boot, taking a snapshot first, verifying afterwards - [Authentication](/config/authentication) — JWT auth, login, refresh, password changes, and auth status - [OIDC / SSO](/config/oidc) — single sign-on with Keycloak, Authentik, Authelia, Google, Azure AD - [WOPI (Office Editing)](/config/wopi) — Collabora Online / OnlyOffice integration diff --git a/docs/config/thumbnail-migration.md b/docs/config/thumbnail-migration.md new file mode 100644 index 00000000..c9c762a7 --- /dev/null +++ b/docs/config/thumbnail-migration.md @@ -0,0 +1,153 @@ +# Thumbnail migration runbook + +Thumbnails used to live as files under `{STORAGE_PATH}/.thumbnails/`. +They now live in the content-addressed blob store, alongside file +content. This page is for operators upgrading across that change. + +**You do not have to do anything.** The migration runs itself, in the +background, on the first boot after the upgrade. The rest of this page +is for operators who want to verify it, take a safety net first, or +understand what it did. + +## What runs, and when + +Two background jobs, dispatched once at startup and daily thereafter: + +| Job | Migrates | Regenerable if lost? | +|---|---|---| +| `thumb_derived_import` | Thumbnails the server rendered from file content | Yes — the next request re-renders | +| `thumb_attached_import` | Previews a client uploaded (`ext-{file_id}.jpg`) | **No** — there is no render path for these | + +Both import each sidecar into blob storage, read it back to confirm the +copy is byte-identical, and only then delete the original. When the +directory is empty it is removed, and `.thumbnails/` stops existing. + +Startup dispatch is non-blocking — the server is ready immediately and +the migration proceeds behind it. A run interrupted by a restart resumes +from where it stopped, so a large installation finishes over several +restarts rather than starting again each time. + +This is controlled by `OXICLOUD_STARTUP_JOBS`, which defaults to: + +``` +OXICLOUD_STARTUP_JOBS=thumb_derived_import?repair=true,thumb_attached_import?repair=true +``` + +To **import without deleting** — migrate now, inspect, delete later: + +``` +OXICLOUD_STARTUP_JOBS=thumb_derived_import,thumb_attached_import +``` + +The sidecars then stay on disk. Trigger the deletion when you are ready +from **Admin → Jobs**, using each job's Repair action. + +To disable startup jobs entirely, set the variable to an empty value. + +## Taking a safety net first + +Recommended for any installation where the uploaded previews matter, and +cheap enough to be worth it regardless. Both parts must be captured +together — a database that references blobs a storage snapshot predates +is worse than neither. + +**1. Stop the server.** A snapshot taken while writes are in flight can +catch a blob that exists on disk without its database row, or the +reverse. + +```bash +systemctl stop oxicloud # or: docker compose stop oxicloud +``` + +**2. Snapshot the database.** + +```bash +pg_dump --format=custom --file=oxicloud-preflight.dump "$DATABASE_URL" +``` + +Use `--format=custom`; restoring it needs `pg_restore --disable-triggers`, +because the folder table carries a self-referencing foreign key that a +plain SQL restore cannot order correctly. + +**3. Snapshot the storage directory.** At minimum `.thumbnails/`, which +is what the migration touches: + +```bash +tar -czf oxicloud-thumbnails-preflight.tar.gz -C "$STORAGE_PATH" .thumbnails +``` + +A whole-directory snapshot is better if you have the space — filesystem +or volume snapshots (ZFS, LVM, EBS) are ideal, since they are atomic and +near-instant: + +```bash +zfs snapshot tank/oxicloud@preflight +``` + +**4. Start the server.** The migration begins in the background. + +Keep both snapshots until you have run the verification below and are +satisfied. + +## Verifying the migration + +Two checks, both from **Admin → Jobs** or the API. Run them after the +migration reports no remaining work. + +**1. Every mapping points at a blob that exists.** Run +`satellites_consistency`. It walks both thumbnail tables and reports any +row whose blob or source is gone. A clean run means nothing was lost in +the bookkeeping. + +``` +POST /api/admin/jobs/satellites_consistency/trigger +``` + +**2. Every blob still hashes to what it claims.** Run +`backend_consistency` with `?deep=true`. It reads every blob back from +storage and re-hashes it, which covers the migrated thumbnails along +with everything else. This is a full read of your storage and can take +hours on a large installation — schedule it accordingly. + +``` +POST /api/admin/jobs/backend_consistency/trigger?deep=true +``` + +A clean pass on both means the thumbnails are readable, correctly +referenced, and byte-intact in their new home. At that point the +snapshots can be discarded. + +## Checking it finished + +`.thumbnails/` is gone. That is the whole test: + +```bash +ls -d "$STORAGE_PATH/.thumbnails" # No such file or directory +``` + +If you instead find `.thumbnails.migrated/`, the migration completed but +could not remove the directory, because something that is not a +thumbnail was inside it — a `.DS_Store` from macOS Finder is the usual +culprit. The tree was moved aside instead of deleted. Its contents are +no longer used and it is safe to remove by hand once you have looked at +what is in there. + +While either directory is absent, the server skips the legacy read path +entirely, at no cost. While `.thumbnails/` is present, reads fall back +to it on a miss, which is what makes the migration invisible to users +while it runs. + +## If something looks wrong + +Every deletion is written to the audit log, naming the job, the file +removed and the blob that replaced it. To review what a migration +removed: + +```bash +journalctl -u oxicloud | grep sidecar_deleted +``` + +A sidecar is only ever deleted after its replacement has been read back +and compared byte-for-byte, so a file that failed that check is still on +disk. Those show up as findings on the job's run in **Admin → Jobs**, +with the reason recorded per file. diff --git a/docs/plan/derived-blobs.md b/docs/plan/derived-blobs.md index 2cb6c2cc..d3133f94 100644 --- a/docs/plan/derived-blobs.md +++ b/docs/plan/derived-blobs.md @@ -339,6 +339,97 @@ Same trims as `content_derived_blobs` — no `size`, no `format`, no Generic naming rather than `file_previews` because the family is real, and each member would otherwise be a new table plus a new `BlobReferenceSource` plus a new term in the consistency recompute. + +### Negative verdicts — `blob_hash` must be nullable + +*(Resolved 2026-08-27. Supersedes the "`.skip` markers are an open +question" note.)* + +The table says "here is the artifact". It cannot say **"there is +deliberately no artifact"**, and absence of a row is ambiguous — it +means both *never attempted* and *attempted, not worth it*. Collapsing +those destroys the only information a negative cache exists to hold. + +Two live cases, not one: + +* **Transcode not beneficial.** `ImageTranscodeService` can only learn + whether WebP is smaller by doing the full decode + encode. When it is + not, it records a zero-byte `{file_id}.{ext}.skip` marker so the next + GET does not repeat the work. +* **Thumbnail not renderable.** `generate_and_persist` returns empty + `Bytes` — "moka's zero-weight negative-entry convention" — for sources + over `MAX_DECODE_PIXELS` (50 MP) or that fail to decode. This one is + **RAM-only**: after moka evicts, a 60-megapixel upload has its full + decode attempted again, forever. + +So this is not a transcode quirk. The rule generalises to every `kind`: + +> **Any derivation whose failure is deterministic in the source content +> is worth memoising negatively.** + +**The discriminator, or the table fills with noise.** Persist a negative +only when it is *both* expensive to compute *and* deterministic in the +content. `can_transcode(mime)` and "not an image" are cheap metadata +checks — recomputing is free and a row would be pure overhead. It is the +ones that cost a decode that earn a row. + +**Permanent vs transient is the load-bearing split.** "Deterministic" +above means *a property of the content*, not merely *a failure that +happened*: + +| Permanent — cache it | Transient — never cache it | +|---|---| +| decode failed (corrupt / unsupported) | generation timeout | +| over `MAX_DECODE_PIXELS` | decode semaphore closed | +| transcode result not smaller | blob read I/O error, OOM under load | + +**Today's code cannot tell them apart, and that must be fixed before any +of this is persisted.** `generate_and_persist` collapses *every* error +into empty `Bytes` — timeouts and semaphore closures included. That is +survivable while the sentinel lives only in moka, which evicts; write +the same signal to the database and a thumbnail that timed out once +under load is marked unrenderable **forever**. So the renderer must +return a typed outcome — rendered / permanently-unrenderable / transient +failure — and only the middle one earns a row. + +The asymmetry sets the default. A wrongly-cached transient is silent and +permanent; a not-cached permanent merely costs repeated work. **When in +doubt, do not cache** — treat unclassified errors as transient. + +**Representation: nullable `blob_hash`.** A sentinel hash was considered +and rejected — it stops `blob_hash` naming a real blob, and every future +reader has to know the lie. NULL says what is true. Costs: + +* `ContentDerivedReferenceSource` needs `AND blob_hash IS NOT NULL`; a + row holding no blob holds no reference. +* The dangling-derived check (row 9 of the coverage matrix) needs the + same guard, or every negative verdict reports as a broken row. +* The `NOT NULL` constraint is dropped. + +**No TTL, and no renderer-version term either.** Both were considered. +Time is the wrong axis: the verdict is deterministic in +`(content, encoder)` and does not decay, so a TTL re-attempts an OOMing +decode on a schedule — reintroducing the exact waste the negative +exists to prevent — while still leaving staleness for most of the window +after a deploy. + +What makes the mechanism unnecessary is that **negative rows are +disposable**: they hold no data, so discarding one costs only a +re-derivation. Upgrading the image library invalidates them with a line +in the same migration as the dependency bump — + +```sql +DELETE FROM storage.content_derived_blobs WHERE blob_hash IS NULL; +``` + +— which beats a version term that must be remembered and leaves dead +rows behind when bumped. Write this down, or someone later builds the +expiry logic this paragraph exists to prevent. + +Same shape, outside this table and not solved here: +`blob_extracted_text` (no extractable text) and `faces.faces` (no faces +detected) are both deterministic negatives currently indistinguishable +from "never processed". With `kind` it's a one-line `ALTER … CHECK`: | Kind | Why it lands here | @@ -858,8 +949,8 @@ Findings each job reports today, and where the new tables land: | # | Edge | Direction | Mechanism | Status | |---|---|---|---|---| | 1 | backend → `storage.blobs` | orphan bytes | `orphan_blob` (backend_consistency) | ✓ | -| 2 | `storage.blobs` → backend | missing bytes | `blob_missing_from_backend` | ✓ | -| 3 | chunk bytes | corruption | `blob_corrupted`, `blob_unreadable` | ✓ | +| 2 | `storage.blobs` → backend | missing bytes | `blob_missing_from_backend` (backend_consistency) | ✓ | +| 3 | chunk bytes | corruption | `blob_corrupted`, `blob_unreadable` (backend_consistency, `?deep=true`) | ✓ | | 4 | manifest → chunks | chunk reaped | `chunk_missing` (files_consistency) | ✓ | | 5 | `files` → Blob | dangling | `missing_blob` (files_consistency) | ✓ | | 6 | `storage.blobs.ref_count` | recompute | `refcount_mismatch` | ✓ chunk level only | @@ -949,6 +1040,74 @@ the request. Read order: backend stack, which is where the disk cache lives. 4. Generate only if step 2 found no row. +### HTTP ETag — move to the derived hash when the order flips + +Shipped ahead of this plan (2026-08-24): the thumbnail ETag is +`"thumb-{source_hash}-{size}-{format}"` on both the REST and the +NextCloud preview endpoint. It replaced a `file_id`-keyed ETag that, +combined with `Cache-Control: immutable`, meant replacing a file's +content never invalidated the client's copy — `file_id` survives the +replacement, so the ETag did too, for a year. + +**That key is still one term short.** A thumbnail is a function of +`(source bytes, size, format, RENDERER)`. Change the encoder, a quality +setting, or EXIF-rotation handling, and identical inputs produce +different bytes under an unchanged ETag — the same staleness class, one +level down. It bites when an already-cached thumbnail is re-rendered +after a renderer change (sidecar evicted, regenerated on miss). + +**The fix is to key on the derived blob's own hash** — the ETag then +*is* the hash of the bytes served, so any change in output invalidates +by construction, with no version constant to remember to bump. It is +self-consistent for free: `store_derived_blob` is +`ON CONFLICT DO NOTHING`, so a re-render never displaces the stored +row, and the ETag therefore always equals what the derived tier will +serve. + +**It must land with the read-order flip, not before.** Today the +derived tier is deliberately read LAST, so an ETag naming the derived +hash would describe a tier the response probably did not come from. +The two agree at creation — `render_and_persist_all_webp` writes both +from the same bytes — but diverge if a sidecar is re-rendered while the +derived row stays pinned by `DO NOTHING`. Sidecar is served, ETag +describes the other one: an ETag that lies about the body is worse than +one that is merely coarse. Two further reasons it has to wait: the +derived tier is WebP-only (`variant = size.dir_name()`, no format in +the key), so non-WebP clients have no row to key on; and nothing +predating this work has a row until `derived_import` backfills. + +So at step 10, alongside the flip: ETag becomes the derived hash, with +the current `source_hash` form kept as the fallback for a variant not +yet generated and for formats the derived tier does not hold. The +`LEFT JOIN` in step 2 above already returns the derived hash in the +same query, so the ETag costs no extra round-trip. + +**Attempted early (2026-08-26) and reverted — the constraint above is +load-bearing.** The attached half shipped and is correct, because an +upload writes its row synchronously before any read can observe it. The +*derived* half was brought forward at the same time and had to be backed +out: the ETag is computed **before** the body, so on a cache miss no row +exists and the handler emits the source-keyed form — then rendering +creates the row, and the very next request resolves to the derived hash. +The validator changed as a side effect of producing the body, so every +first render was immediately stale. Caught by +`thumbnail_etag_content_keyed.hurl`, where two consecutive GETs of an +unchanged file stopped revalidating to 304. + +**The flip alone does NOT remove the hazard** *(corrected 2026-08-26 — +an earlier revision of this paragraph claimed it did)*. A first render +still creates the row as a side effect of producing the body, whatever +the read order, so two consecutive reads would still straddle its +appearance. + +What actually removes it is resolving the ETag **after** generation on +the 200 path. A 304 can only fire when the client already holds a +validator, which means it has been served before, which means the row +exists — so the *conditional* path can safely consult the derived hash +up front, while the *generating* path computes it from bytes it now +holds. That is a handler restructure, not an ordering change, and it is +the actual prerequisite for the derived-hash ETag. + **The disk cache is `CachedBlobBackend`, reused unchanged.** No thumbnail-specific cache, no second root path. Routing derived blobs through the same stack gets, for free: @@ -1109,8 +1268,101 @@ filesystem and a remote backend for hours). It sweeps the cold tail: reference forever. - Reports imported / skipped-orphan / skipped-jpeg / failed counts. -**Phase 3 (release N+1): delete** the fallback module and the sidecar -directories, gated on the job reporting an empty tail. +**Phase 3: the job deletes, not a release.** *(revised 2026-08-26 — +supersedes "delete in release N+1, gated on an empty tail")* + +Sidecars are **local disk**. A release cannot know whether every +instance has drained, so gating deletion on "the tail is empty" asks an +operator to coordinate a fact nothing reports — and there is no way to +know when, or whether, they will trigger the jobs at all. Instead the +import unlinks each sidecar it has successfully imported, so **each +instance drains itself** and the directory becomes removable once +genuinely empty. + +Three constraints on that: + +- **Verify readback before unlinking.** Import → read the derived blob + back through the normal stack → *then* delete. A store that reported + success but landed unreadable would otherwise take the last copy with + it. Cheap next to the decode already performed, and it is the + difference between a migration and a data-loss bug. +- **Only after the read-order flip.** Deleting while the sidecar is + still read *first* sends reads to the derived tier as a side effect of + the migration — its first production traffic arriving by accident + rather than by decision. +- **Opt-in** (`?delete_imported=true`). A migration that deletes on its + default setting is surprising, and it is the same instinct as + no-silent-auto-repair: early runs import only, so an operator can + inspect before committing. + +Register it as a **scheduled tick**, not a boot-time trigger: it is +idempotent and resumable, so periodic is safe, whereas walking a large +`.thumbnails/` during startup delays readiness for nothing. + +**Phase 4: the fallback disables itself.** *(revised 2026-08-29 — +supersedes "the only remaining release is removing the fallback read +path")* + +Removing the fallback in a release has the same flaw as gating deletion +on an empty tail, one level up: sidecars are local disk, so no release +can know that every instance has drained. Ed's framing is the answer — +the only removal you can actually write is `if the tier is gone, return`. + +So `ThumbnailService::initialize` probes the size directories once at +boot and stores the result. When absent, every fallback read +short-circuits on a relaxed atomic load, no syscall. The code stays, +costs nothing, and can be deleted whenever — or never. No coordination, +no named release. + +Two things had to change for absence to be reachable at all: + +- **`initialize` no longer creates the directories.** It + `create_dir_all`-ed all three at every boot, so the job removed them + and the next restart put them back; the absence this gates on was + unreachable by construction. Nothing has written a sidecar since step + 10d2, so there was nothing to create them for. +- **The probe tests the size directories, not the root.** On macOS + Finder leaves a `.DS_Store` in the root, which blocks `remove_dir` + there permanently. Gating on the root would keep the fallback alive on + every developer machine for a reason unrelated to thumbnails. No size + directory means no sidecar. + +The root removal now reports its outcome instead of discarding it — +it is the one result an operator is waiting for, and `.DS_Store` is a +failure worth naming rather than a silent no-op. + +### Prerequisite: one persist function (found 2026-08-26) + +**Four render paths write a sidecar; only one also writes the derived +row.** `store_derived_blob` has a single call site — in +`render_and_persist_all_webp` — while `fs::write(&thumb_path, …)` has +five. `get_thumbnail`, `generate_and_persist` and +`generate_all_sizes_background` all persist sidecar-only. + +That breaks the migration's premise rather than merely being untidy: an +on-demand render (cache miss, a size never generated, an evicted +sidecar) keeps producing un-migrated state *after* the import runs, so +the tail never empties and the deletion gate never opens. + +So before the imports can converge, all render paths must go through +**one** `persist_thumbnail` that writes the sidecar, the derived blob +and the moka entry together — the same single-source move as +`storage.copy_file_satellites`. What it writes then becomes a policy in +one place, so "stop writing sidecars" is later a one-line change rather +than four edits. + +Interim setting is **dual-write**, for two reasons: it is what makes the +backlog finite, and it leaves reads untouched while the derived tier is +still unproven. Cost is one extra local `fs::write` per render, +negligible beside the decode. Note the sidecar *read* path must survive +until the directories are empty regardless, so stopping the write early +buys nothing. + +Cost to be aware of: `ThumbnailService` holds no `DedupService` — it is +a per-call parameter (`dedup: Option<&DedupService>`) — so the +consolidation threads it through those paths, and +`generate_and_persist` takes a `thumb_path` where it will need the +`blob_hash` instead. ### The "just delete it" opt-out is no longer universally safe @@ -1157,7 +1409,35 @@ hardcoded SQL). New sources bolt on independently. after, step 5 lands at scale; per-row HEADs do not survive a 4× row count. 7. **`ImageTranscodeService`** — same shape, `kind = 'transcode'`, - no new table. + no new table. **Scoped 2026-08-27, not started.** The service does + not currently write the derived tier at all, which is the same gap + `persist_rendered` closed for thumbnails, and it must be closed + before `transcode_import` can converge. + + Three concrete pieces, in order: + + * **Thread the source hash to the call site.** `get_transcoded` takes + `file_id` only, and the table is content-keyed. The hash IS + available one frame up — `file_retrieval_service::try_transcode` + is called from a scope holding `dto.content_hash` — so it is a + parameter to add, not a lookup to invent. Do **not** hash + `original_content` on the fly: that is a BLAKE3 over the whole + file on every request. + * **Give the service a `BlobHandler`.** It has no field for one + (`cache_dir`, `memory_cache`, `stats`), so this is a constructor + and DI change — mind the construction order, as + `ThumbnailService` hit the same thing and solved it with a + per-call parameter instead. + * **Write the `.skip` markers as negative rows** — resolved: nullable + `blob_hash`, see *Negative verdicts*. Thumbnails need the same + treatment for undecodable and over-`MAX_DECODE_PIXELS` sources, + which are RAM-only today, so do both together rather than twice. + + Note the cache is keyed `{file_id}:{ext}` in memory and + `.transcoded/{ext}/{file_id}.{ext}` on disk, so it also carries the + file-vs-content keying mismatch that `transcode_import` has to + re-key. Fixing the write path first means the import only has to + handle history, not a moving target. 8. **`storage.copy_file_satellites` consolidation** — collapse the two copy paths onto one helper, with a manifest-aware reference bump. **Blocks step 9**: adding a file-keyed table before this means @@ -1169,9 +1449,101 @@ hardcoded SQL). New sources bolt on independently. table. Lands with step 5. File-keyed, never in `content_derived_blobs`. Register it in `copy_file_satellites` and declare its version semantics. -10. **`derived_import` job + the dual-read fallback** — see the - migration section. Phase 3 (deleting the fallback and the sidecar - dirs) is a separate later release, gated on an empty tail. +10. **Import jobs + the dual-read fallback** — see the migration + section. Revised ordering as of 2026-08-26: + + a. **Consolidate onto one `persist_rendered`** (dual-write) — **done + 2026-08-26**. Was the blocker: only one of four render paths wrote + the derived row, so the import could never converge. Every live + render now dual-writes. Two paths still pass `None` and stay + sidecar-only, which is safe *only* because both are reachable + solely through the `ThumbnailPort` impl and nothing holds a + `dyn ThumbnailPort` — if either gains a real caller it must take a + `DedupService` first. See *Prerequisite: one persist function*. + b. **`thumb_derived_import`** (shipped) and **`thumb_attached_import`** + (shipped) — two jobs, not one, because the keying differs and that + difference is the security boundary. A third, `transcode_import`, + is still needed: `ImageTranscodeService` **already exists** and + caches `.transcoded/{ext}/{file_id}.{ext}`, so those must be + **re-keyed** file→content on import (legitimate only because a + transcode is derivable). Its `.skip` markers import as **negative + rows** with a NULL `blob_hash` — see *Negative verdicts* — so the + verdict survives the deletion of `.transcoded/`, which it + otherwise would not. + + **Not next, and deliberately so.** Two prerequisites, both learned + the hard way on the thumbnail side: + + * **Format must move into `variant` first.** Transcodes are + inherently multi-format, and `variant` is keyed on size alone — + the same gap that keeps JPEG thumbnails on the sidecar (see + 10c). Importing before that means migrating into a schema that + cannot hold the data without collisions. One migration unblocks + both. + * **Step 7 before the import.** `ImageTranscodeService` writes only + its file-keyed cache today, so an import would run against a + cache still growing and never reach an empty tail — precisely + the trap `persist_rendered` had to close for thumbnails. + + Order: format-in-`variant` → step 7 → `transcode_import`. + c. **Flip the read order**, derived first — **done 2026-08-26**. Two + things it is not: a two-line swap, and an ETag fix. + + A derived miss must **fall through** to the sidecar, where the old + code terminated the lookup — while the imports drain, most content + has a sidecar and no row, so terminating would report "no + thumbnail" for nearly everything. + + And it is **WebP-only**. `store_derived_blob` writes `image/webp` + with `variant` keyed on size alone, no format term, so a JPEG + request matches the WebP row and gets the wrong codec — a + regression the old ordering hid, because the `.jpg` sidecar won + first. JPEG clients therefore stay on the sidecar, **and the + sidecar cannot be deleted for them** until `variant` encodes + format. That is a new prerequisite for (e), not a detail: it means + a migration to `(kind, variant, format)` — or a format term inside + `variant` — has to land before the directories can go. + d. **Enable deletion** in the import jobs (opt-in, readback-verified) + — **done 2026-08-27**, both halves, sharing one + `verify_and_unlink`. + + d2. **Stop writing sidecars.** *(Added 2026-08-27 — the sequence + above was missing this, and (e)'s gate is unreachable without + it: dual-write means any render or upload recreates the + directory seconds after the job removes it, so "no longer + exists" can never hold.)* + + Two sides, and they differ in what a failed write costs: + + * **Rendered / content-keyed — safe now.** Delete the `fs::write` + in `persist_rendered`. No gate beyond the read flip, which has + landed. Existing sidecars are untouched, so a box that has not + imported yet keeps its fallback for old content; new content + goes only to the derived tier, which the read path already + prefers. If the derived store fails the bytes are still served + and simply not cached — regenerable by definition. + * **Uploaded / file-keyed — needs a change first.** + `upload_thumbnail_impl` logs and still returns 201 when + `store_attached_blob` fails, which is safe *only* because the + `ext-` sidecar catches it. Remove that sidecar while the store + is best-effort and a user's uploaded preview can vanish + silently behind a success response. These are the + non-regenerable bytes, so **the PUT must fail** before the + write is removed. + + Order matters: make the attached store fatal, *then* drop its + sidecar. Reversed, it trades a silent data-loss window for an + empty directory. + + e. **Remove the fallback read path** once the directory no longer + *exists* — not merely once it is empty. Two reasons. Empty is a + momentary property an on-demand render can undo, whereas absence + is one-way and observable, so the job removes the directory after + draining it and that absence is the proof. And it is far cheaper + to test: existence is a single `stat`, while emptiness costs an + `opendir`/`readdir`/`closedir` — which matters if the fallback + ever gates on it per read rather than once at boot. The only + remaining release, and no data is at stake by then. 11. **`DedupService` → `BlobHandler` rename** — decided, mechanical, 34 files. Standalone commit, `src/AGENTS.md` updated with it. Can land at any point; last is easiest, since every earlier slice @@ -1257,10 +1629,40 @@ Schema rename (deferred, requires migration): - `storage.chunk_manifests` → `storage.blob_manifests` (or keep — arguable) - `BlobStorageBackend` trait → `ChunkStorageBackend` — reads and writes physical chunks, not blobs +- **`blobs_consistency` job → `chunks_consistency`** — it iterates + `storage.blobs`, so it inherits whatever that table is called. -`file.blob_hash` semantics stay — references a Blob via its -manifest OR (for pre-CDC legacy) points directly at a single-chunk -Blob whose hash equals its lone chunk's hash. +### The job rename has two rules of its own + +**Travel with the schema, never ahead of it.** A job named +`chunks_consistency` iterating a table still called `storage.blobs` is +*more* confusing than today's mismatch, not less. + +**Never recycle `blobs_consistency`.** Under the corrected taxonomy the +manifest job *is* the blob-level job, so the freed name looks +available — and reusing it would be the worst outcome available. A job +name that survives a release while changing meaning silently breaks +`POST /api/admin/jobs//trigger` URLs, every historical row in +`background_runs.job_name`, and any dashboard or alert keyed on it. +`manifests_consistency` is unambiguous under either taxonomy; leave it +alone. Net effect: one job renamed, not two swapped. + +Budget for the operational cost either way — job names are not internal +identifiers. A rename orphans past runs unless `background_runs.job_name` +is migrated alongside, and any runbook naming the old one breaks. Worth +an alias period or an explicit release note. + +### Explicitly NOT renamed + +- **The `.blob` on-disk suffix** (`.blob` in `LocalBlobBackend` + and `CachedBlobBackend`). Correcting it to `.chunk` would mean + renaming every file in every deployment's blob store — a migration + whose cost is wildly out of proportion to the clarity gained, and one + that can fail halfway. The suffix is an implementation detail no + consumer parses; leave it. +- **`file.blob_hash`** — semantics stay. It references a Blob via its + manifest OR (for pre-CDC legacy) points directly at a single-chunk + Blob whose hash equals its lone chunk's hash. Scope for this rename: ~23 files touch the SQL, plus a migration for the table rename. Not free. Ship AFTER the tier-2 write-side diff --git a/docs/plan/job-registry.md b/docs/plan/job-registry.md index ebc30e6e..b9845953 100644 --- a/docs/plan/job-registry.md +++ b/docs/plan/job-registry.md @@ -171,6 +171,58 @@ Native services implement this trait on an existing service type (no new wrapper) and register a single `Arc` with the scheduler. +### Self-description — `description` / `mutates` / `repair_description` + +Three defaulted methods on both `JobHandler` and `RecoverableJobHandler` +let a job tell the admin UI what it is. `RecoverableAdapter` forwards +them, since the registry only ever holds `dyn JobHandler`. + +```rust +fn description(&self) -> &'static str { "" } +fn mutates(&self) -> Mutates { Mutates::Never } +fn repair_description(&self) -> Option<&'static str> { None } + +pub enum Mutates { Never, Always, OnRepairOnly } +``` + +They surface on `JobSummary` (`GET /api/admin/jobs`) and drive the +panel: `Never` earns a read-only badge and triggers straight through, +`Always` confirms first, `OnRepairOnly` is safe to run and confirms only +when the repair variant is picked. `repair_description.is_some()` is +what renders the repair toggle at all, and its text is the confirmation +copy. + +**Why three values and not a boolean.** A job can be read-only by +default and destructive under `?repair=true`; a boolean has to answer +wrongly for one of those two modes, and `false` on something that +deletes files is the dangerous direction to be wrong in. It is also +where the recovery framework is heading — discovery-only default, +mutation behind an opt-in — so a tenant that later grows a repair arm +changes this one value and nothing else. + +**Why `Option<&str>` and not `supports_repair: bool` + prose.** +Presence gates the toggle, content supplies the wording. Split across +two methods they can disagree; and the frontend cannot invent the +wording itself, because correcting a counter and unlinking files off +disk are not the same warning. The two are independent, not derived +from each other: the thumbnail imports are `Always` *and* +repair-capable. + +`OnRepairOnly` with no `repair_description` is rejected at registration +— it claims to mutate only under a flag it does not support, and would +render as safe with no reachable mutating path. + +**Why English in the trait, not `locales/*.json`.** A description that +lives away from the behaviour rots the moment a job changes, invisibly, +and a translator cannot know what `manifests_consistency` reconciles. +i18n can layer on later keyed by job name with these as the fallback, +matching the frontend's `t(key, params, fallback)` — a missing +translation then degrades to English from code rather than to a blank +panel. No rework needed to get there. + +Defaults exist so the methods could be added without touching every +job at once; every registered job declares all three today. + ### `JobOutcome` ```rust @@ -685,6 +737,98 @@ trigger) resumes any `Paused` row per the normal flow. Consistency-check.md's existing consistency-scoped sweep collapses into this general one. +### Startup jobs — `OXICLOUD_STARTUP_JOBS` + +A comma-separated list of jobs to dispatch once, in the background, +after the scheduler is ready. Each entry is a registered job name, +optionally with the same query syntax the admin trigger URL uses. + +**The default is both migration jobs, in repair mode:** + +``` +OXICLOUD_STARTUP_JOBS=thumb_derived_import?repair=true,thumb_attached_import?repair=true +``` + +An explicit value replaces that list; an empty value disables startup +jobs entirely. + +**Why it exists.** Scheduled ticks deliberately never pass `repair` — a +job that deletes on its default setting is what no-silent-auto-repair +forbids. But that left the migration jobs unable to finish on their +own: a deployment whose operator never opens the admin panel re-imports +sidecars it already imported, forever, and never drains the directory. + +**Why the default deletes anyway.** Relying on operators to edit `.env` +has the same failure mode one level up — the ones who never edit it are +exactly the ones whose migration never completes. So this is a +deliberate exception to no-silent-auto-repair, and it rests on three +properties that must keep holding: + +- **Nothing is deleted before its replacement has been read back.** + `verify_and_unlink` imports, reads the blob back through the normal + stack, and only then unlinks; a store that reported success but landed + unreadable keeps its sidecar. This matters most for + `thumb_attached_import`, whose bytes are user-uploaded previews with + no render path — a wrong deletion there is permanent, where a wrong + deletion of a server-rendered thumbnail costs a re-render. +- **Sidecars whose source is gone are deleted without a readback**, + because there is nothing to read back and nothing can reference them + again. Unrecoverable and unreachable are different things; these are + both. +- **Every deletion is audited**, so what a boot removed, and from which + source, is reconstructable afterwards. + +The consequence to hold in mind: an upgrade deletes on first boot, in +every deployment at once, with no operator action. A regression in the +readback path would be simultaneous and unrecoverable, so that code is +load-bearing. Operators who want to inspect before committing set +`OXICLOUD_STARTUP_JOBS=thumb_derived_import,thumb_attached_import` — +same jobs, import only. + +It is not a "run everything in repair mode" switch. Each job is named +individually and carries its own flags. + +**Validation is fail-fast.** An unknown job name panics at boot — the +registry is fully populated by then, so a name that doesn't resolve is a +typo or a stale rename, and ignoring it would leave a migration that +silently never runs. Unknown flags panic too: a dropped `?repare=true` +would leave the job in discovery-only mode while the operator believed +the tier was draining, and the symptom ("it never finished") surfaces +months later with nothing pointing back at the config. + +**Dispatch is non-blocking.** `tokio::spawn`, so readiness never waits +on a job that may walk a filesystem for hours. Jobs in the list run +sequentially within that task, not concurrently: they contend for the +same directories and pool, and the exclusivity gate would turn overlap +into a *skipped* run rather than a queued one. + +**Interrupted runs resume.** The boot recovery sweep above runs first +and flips every abandoned `Running` row to `Paused` with its cursor +intact; `run_or_resume` then picks Resume over a fresh start. So a +migration killed by a restart continues where it stopped, and completes +across however many restarts it takes. + +That is a deliberate exception to "do NOT auto-resume" — scoped to the +named jobs only. The rule protects against a restart silently resuming +work nobody asked for; here somebody did ask, in configuration, and not +having to ask again is the entire point. Every other paused run still +waits for an operator. + +A resumed run keeps the flags it started with (`repair` / `deep` are +persisted to `params` on the fresh open and read back on resume), so +editing the config mid-migration does not retroactively change a run +already in flight. + +**Safe to leave set.** Each job is idempotent and resumable; once the +tier has drained, a run is a `read_dir` over three directories that +returns nothing — and after the directory is removed, not even that. + +**Visible in the admin panel.** These are ordinary registered jobs: +they appear in `GET /api/admin/jobs`, are triggerable by hand, and +record the same runs and findings. Rows named here additionally carry a +`startup` object with the configured flags, so an operator can see that +a job deletes files on every boot rather than only when someone clicks. + ### Admin surface (recoverable runs) Same URL taxonomy as Part 1 — resource-first, action second, all diff --git a/docs/plan/storage-key-rotation.md b/docs/plan/storage-key-rotation.md index 7b908f3b..2f4a18df 100644 --- a/docs/plan/storage-key-rotation.md +++ b/docs/plan/storage-key-rotation.md @@ -179,7 +179,17 @@ Guardrail: head pair's fingerprint differs from the second entry's — signals "you added new keys but haven't rotated legacy blobs yet". * A **legacy-blob counter** is surfaced in the admin panel per storage entry. - The counter is maintained by `blobs_consistency`: during its normal walk it + + > **Retarget (post-split).** This plan names `blobs_consistency` as the + > host for the magic-byte branch throughout. That is no longer the right + > tenant: `blobs_consistency` became database-only and opens no backend, + > while `backend_consistency` owns every physical check and already holds + > the enumeration. Read `backend_consistency` wherever the sections below + > say `blobs_consistency`. Nothing else about the design changes — the + > magic-byte check still rides along on an existing walk, and still lands + > in the run's `stats` bag. + + The counter is maintained by that scan: during its normal walk it branches on the magic-byte check and records the legacy count as a run statistic on `jobs.recoverable_runs` (existing surface, no schema hit). The admin panel reads the most recent count and displays it. Refresh cadence is diff --git a/docs/plan/storage-multi-entry.md b/docs/plan/storage-multi-entry.md index 00c91046..c10c429c 100644 --- a/docs/plan/storage-multi-entry.md +++ b/docs/plan/storage-multi-entry.md @@ -407,10 +407,12 @@ Per slice, plus these end-to-end scenarios in Hurl: 6. **In-place encryption rotation refused**: two entries, same S3 bucket, different encryption keys. Trigger migration → refuses with the specific error message pointing at the encryption case and the two-step workaround. -7. **`?storage=` on blobs_consistency**: run against `s3_prod` before +7. **`?storage=` on backend_consistency**: run against `s3_prod` before cutover. Full walk, `probed_storage` in run row. Then cutover, then rerun against `local_main` — verifies old backend still has everything. -8. **Unknown storage name**: `POST /admin/jobs/blobs_consistency/trigger?storage=nope` + (This scenario named `blobs_consistency` until that tenant became + database-only; entry scoping belongs to whichever job opens a backend.) +8. **Unknown storage name**: `POST /admin/jobs/backend_consistency/trigger?storage=nope` → 400 with known-names list. No run row created. 9. **Missing entry at boot**: `active_backend_name = "gone"` but `_ENTRIES` doesn't include it → boot aborts with the specific message pointing at diff --git a/example.env b/example.env index 6c5baf05..9fa80904 100644 --- a/example.env +++ b/example.env @@ -55,6 +55,36 @@ OXICLOUD_SERVER_HOST=0.0.0.0 # Recommended: 127.0.0.1:9090 with node_exporter-style scrapers. #OXICLOUD_METRICS_LISTEN=127.0.0.1:9090 +# ── Startup jobs ────────────────────────────────────────────────────── +# Background jobs dispatched once, after the scheduler is ready. +# Comma-separated; each entry is a registered job name, optionally with +# the same flags the admin trigger URL takes (force, deep, repair, +# storage). +# +# DEFAULT (applied when this variable is unset): +# thumb_derived_import?repair=true,thumb_attached_import?repair=true +# +# Those two migrate thumbnails out of the legacy .thumbnails/ directory +# into blob storage and then delete the originals, so the migration +# completes without anyone having to trigger it from the admin panel. +# Each sidecar is read back through the normal stack before it is +# unlinked, and every deletion is written to the audit log. +# +# Dispatch is non-blocking — startup never waits on a job. A run +# interrupted by a restart resumes from its cursor on the next boot, so +# a long migration finishes across restarts. Safe to leave at the +# default: the jobs are idempotent, and once the directory is drained a +# run does nothing at all. +# +# An unknown job name or flag is a FATAL error at boot, not a warning — +# a silently ignored entry means a migration that never runs. +# +# To disable every startup job, set this to the empty value: +#OXICLOUD_STARTUP_JOBS= +# +# To import without deleting (inspect first, delete later by hand): +#OXICLOUD_STARTUP_JOBS=thumb_derived_import,thumb_attached_import + # ── Upload size caps ────────────────────────────────────────────────── # See docs/config/storage-fine-tuning.md for sizing guidance. diff --git a/frontend/src/lib/api/types.ts b/frontend/src/lib/api/types.ts index 9c967495..b3f44d80 100644 --- a/frontend/src/lib/api/types.ts +++ b/frontend/src/lib/api/types.ts @@ -618,8 +618,32 @@ export interface PausedRunBrief { total?: number; } +/** + * When a job changes state — `RecoverableJobHandler::mutates()` on the + * backend. Three values rather than a boolean because the interesting + * case is conditional: a job can be read-only by default and destructive + * under `?repair=true`. + * + * - `never` — read-only under every flag. Render a read-only badge; no + * confirmation needed to trigger. + * - `always` — changes state on a plain run. Confirm before triggering. + * - `on_repair_only` — safe to trigger; confirm only when the repair + * toggle is on. + */ +export type Mutates = 'never' | 'always' | 'on_repair_only'; + export interface JobSummary { name: string; + /** One or two sentences on what the job does, in English, authored + * next to the handler. Absent for jobs that haven't declared one — + * omit the line rather than rendering an empty block. */ + description?: string; + mutates: Mutates; + /** Present iff `?repair=true` does something beyond a default run; + * describes what it ADDS. Presence is what gates the repair toggle; + * the text is the confirmation copy. Independent of `mutates` — the + * thumbnail import jobs are `always` AND repair-capable. */ + repair_description?: string; interval_ms?: number; next_run_at?: string; last_run_at?: string; @@ -639,6 +663,19 @@ export interface JobSummary { * for this job. Distinct from `running` — a paused run is * resumable via the same trigger endpoint. */ paused_run?: PausedRunBrief; + /** Present iff `OXICLOUD_STARTUP_JOBS` names this job — the flags it + * is dispatched with at every boot. Worth showing: a job configured + * with `repair: true` deletes on every restart, and the row would + * otherwise suggest that only happens when someone clicks Run. */ + startup?: StartupTrigger; +} + +/** Flags a job configured in `OXICLOUD_STARTUP_JOBS` runs with. */ +export interface StartupTrigger { + force: boolean; + deep: boolean; + repair: boolean; + storage?: string; } /** diff --git a/frontend/src/lib/components/AdminJobsPanel.svelte b/frontend/src/lib/components/AdminJobsPanel.svelte index d247ba58..6c51e637 100644 --- a/frontend/src/lib/components/AdminJobsPanel.svelte +++ b/frontend/src/lib/components/AdminJobsPanel.svelte @@ -167,7 +167,7 @@ .slice() // `consistency_batch` is served by the top-bar // action buttons; hiding it here removes the - // duplicate table row. `hasBatch` still checks the + // duplicate table row. `batchJob` still reads from the // full fetched list so the top buttons only render // when the coordinator is actually registered. .filter((j) => j.name !== 'consistency_batch') @@ -180,7 +180,7 @@ // Track whether the coordinator is registered so the // top-bar buttons can gate on it without checking `jobs` // (which now filters it out). - hasBatch = fetched.some((j) => j.name === 'consistency_batch'); + batchJob = fetched.find((j) => j.name === 'consistency_batch') ?? null; loadError = null; } catch (e) { loadError = errorMessage(e); @@ -250,13 +250,15 @@ // ─── Expansion toggles ───────────────────────────────────────────── - function toggleJob(name: string) { - if (expandedJob === name) { + function toggleJob(job: JobSummary) { + if (expandedJob === job.name) { expandedJob = null; } else { - expandedJob = name; - // Lazy-load on first open, refresh on subsequent opens. - void loadRuns(name); + expandedJob = job.name; + // Lazy-load on first open, refresh on subsequent opens. Only + // recoverable jobs have runs to load — the others expand purely + // to show their description. + if (isRecoverable(job)) void loadRuns(job.name); } } @@ -458,8 +460,9 @@ * Per-severity finding counts from `last_outcome.extra.severity_counts` * (a JSON object populated by `run_or_resume`). Missing / older * runs return an empty record — callers should tolerate absent keys. - * The three severity values are the ones consistency tenants emit - * today: `data_loss`, `inconsistent`, `anomaly`. + * Severity values emitted today: `data_loss`, `inconsistent`, + * `anomaly`. The set is open (the column is TEXT), so unknown keys + * must degrade rather than throw. */ function lastSeverityCounts(job: JobSummary): Record { if (!job.last_outcome || job.last_outcome.outcome !== 'ok') return {}; @@ -480,6 +483,13 @@ return (s.data_loss ?? 0) + (s.inconsistent ?? 0); } + /** + * Informational findings. `anomaly` is the wire value; "notice" is + * what the panel calls it — there is no separate `notice` severity. + * A job that acted on what it found (a repair run deleting an + * orphaned sidecar) records the same severity and says so in the + * finding's `detail`. + */ function anomalyFindingCount(job: JobSummary): number { return lastSeverityCounts(job).anomaly ?? 0; } @@ -627,38 +637,75 @@ // Jobs that respect `?deep=true`: // * `consistency_batch` — propagates deep to every child that // understands it - // * `blobs_consistency` — deep mode re-reads + re-hashes every - // blob for silent bit-rot detection (severity `data_loss`). - // Full read of storage; can take hours on big installs — the - // "Run" (normal) button on the same row does the cheap - // existence probes only. + // * `backend_consistency` — deep mode re-reads + re-hashes every + // matched blob for silent bit-rot detection (severity + // `data_loss`). Full read of storage; can take hours on big + // installs — the "Run" button on the same row does the + // enumeration merge-join only. This was `blobs_consistency` + // until that tenant became database-only. function supportsDeep(name: string): boolean { - return name === 'consistency_batch' || name === 'blobs_consistency'; + return name === 'consistency_batch' || name === 'backend_consistency'; } - // Jobs whose handler consults `args.repair` and applies a - // corrective UPDATE against the finding it just emitted. Only the - // two ref_count tenants today; `consistency_batch` also accepts - // the flag (fans out to both) and is surfaced separately as the - // top-bar "Repair ref_counts" button. Keep this list narrow — - // adding a job here without a matching backend handler produces a - // silently no-op button that confuses operators. - function supportsRepair(name: string): boolean { - return name === 'blobs_consistency' || name === 'manifests_consistency'; + // Whether `?repair=true` does anything for this job — declared by the + // handler itself via `repair_description()`, not by a name allowlist + // here. The allowlist this replaces named only the two ref_count + // tenants and silently omitted every repair-capable job added since, + // so the thumbnail imports could not be run in repair mode from the + // panel at all despite supporting it. + function supportsRepair(job: JobSummary): boolean { + return !!job.repair_description; } - async function onTriggerWithRepairConfirm(name: string) { + // What the repair adds, in the handler's own words. The backend owns + // this string precisely because the wording differs per job: correcting + // a counter and unlinking files off disk are not the same warning, and + // the frontend has no way to tell them apart. + async function onTriggerWithRepairConfirm(job: JobSummary) { const ok = await confirmDialog({ - title: t('admin.jobs.run_repair_confirm_title', 'Repair drifted ref_counts?'), - message: t( - 'admin.jobs.run_repair_confirm_body_scoped', - { name }, - 'Runs {{name}} and applies a corrective UPDATE to any counter that disagrees with its live reference count. Content-safe: only counters change; blob content and file rows are untouched.' + title: t( + 'admin.jobs.run_repair_confirm_title_scoped', + { name: job.name }, + 'Run {{name}} in repair mode?' ), + message: job.repair_description ?? '', confirmText: t('admin.jobs.run_repair_confirm', 'Repair'), danger: true }); - if (ok) await onTrigger(name, { repair: true }); + if (ok) await onTrigger(job.name, { repair: true }); + } + + // Confirmation before a plain run of a job that writes. `never` jobs + // trigger straight through — that is the point of the flag — and + // `on_repair_only` jobs are read-only until the repair variant is + // picked, which carries its own confirm. + async function onTriggerGuarded(job: JobSummary) { + if (job.mutates === 'always') { + const ok = await confirmDialog({ + title: t('admin.jobs.run_mutating_confirm_title', { name: job.name }, 'Run {{name}}?'), + message: + job.description || + t('admin.jobs.run_mutating_confirm_body', 'This job changes stored state when it runs.'), + confirmText: t('admin.jobs.run', 'Run'), + danger: true + }); + if (!ok) return; + } + await onTrigger(job.name); + } + + // Row badge. `never` is the one worth stating outright — it is the + // answer to "is it safe to click this on production?", and it is the + // question an operator asks before every trigger. + function mutatesLabel(job: JobSummary): string | null { + switch (job.mutates) { + case 'never': + return t('admin.jobs.mutates_never', 'read-only'); + case 'on_repair_only': + return t('admin.jobs.mutates_on_repair_only', 'read-only unless repaired'); + default: + return null; + } } function isRunning(job: JobSummary): boolean { @@ -679,11 +726,13 @@ // coordinator is registered (should always be true post-Slice 5, // but check defensively so the button doesn't appear on an old // deployment before this component is upgraded). - // Coordinator registration flag — set imperatively in - // `loadJobs` because `jobs` no longer contains the - // `consistency_batch` row (filtered out to avoid duplicating the - // top-bar action buttons). - let hasBatch = $state(false); + // Held as the whole summary rather than a boolean because the + // top-bar buttons need its `repair_description` — the coordinator + // describes its own repair semantics, same as every table row. + // Set imperatively in `loadJobs` because `jobs` no longer contains + // the `consistency_batch` row (filtered out to avoid duplicating + // the top-bar action buttons). + let batchJob = $state(null);
@@ -697,10 +746,12 @@

- {#if hasBatch} + {#if batchJob} + {@const batch = batchJob} - - + + {#if batch.repair_description} + + {/if} {/if} {/if} - {#if supportsRepair(job.name)} + {#if supportsRepair(job)}