diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 7a63df25..24e0b57b 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -23,10 +23,61 @@ ignore = [ "RUSTSEC-2026-0098", "RUSTSEC-2026-0099", + # h2 0.3.27 — RUSTSEC-2026-0258 "unbounded empty DATA frames" + # (GHSA-q83h-524g-xf6h). Transitive via aws-smithy-http-client 1.1.12 + # → hyper 0.14.32 → h2 0.3.27. The patched line is 0.4.16+, but hyper + # 0.14's `h2 = "0.3"` requirement pins us to the 0.3.x branch which + # will not receive a backport — real fix requires aws-smithy-http-client + # to migrate to hyper 1.x (which our other h2 copy — 0.4.16, already + # bumped — is on). The 0.4.x copy is fixed via `cargo update`; this + # ignore covers only the 0.3.x chain. + # + # Severity: low (advisory's own classification). Attack is empty-DATA- + # frame flooding by a malicious HTTP/2 peer → memory pressure or panic. + # In this codebase h2 0.3.x runs strictly on the CLIENT side of AWS + # SDK requests to S3 endpoints. Exploitation requires either + # compromising AWS S3 (out-of-scope) or MitM with a valid TLS cert + # for the configured S3 host (bigger problem than the DoS). No + # data-integrity or auth impact; panic path contained by + # request-level unwind. + # + # Un-ignore trigger: aws-smithy-http-client releases a version that + # switches to hyper 1.x (checkable with `cargo tree -i h2@0.3` — the + # command returns no rows once the chain is gone). Track upstream at + # https://github.com/smithy-lang/smithy-rs/issues (search "hyper 1"). + "RUSTSEC-2026-0258", + # instant unmaintained — transitive via azure_core 0.21.0 (latest available). # No direct security impact; no upgrade path exists. "RUSTSEC-2024-0384", + # azure_core 0.21.0 writes the `authorization` header value to logs — + # `policies/transport.rs` does `debug!("…{request:#?}")` over the whole + # request. For a SharedKey entry that value is the per-request HMAC + # signature; for a SAS entry it is the token. Severity 6.5 (medium). + # + # The advisory says "upgrade to >=0.22.0". That version does not exist: + # `azure_core` jumped 0.21 → 1.x, and `azure_storage_blobs` never left + # 0.21.0 before being archived. So the stated remedy IS the official-SDK + # migration, tracked separately alongside the quick-xml pair above. + # + # Not reachable at our log levels: the line is `debug!` on the + # `azure_core::policies::transport` target, and the default filter is + # `info`. It fires only if an operator explicitly asks for + # `RUST_LOG=…,azure_core=debug`, which is not hypothetical — that is the + # invocation used to diagnose the Azurite migration hang. **Do not run + # `azure_core=debug` against a real Azure account**; it prints request + # signatures to the terminal. Against Azurite it only exposes the + # published dev key's signatures. + # + # A subscriber-level directive pinning that target off was prototyped + # and rejected 2026-09-02 — not worth carrying a filter hack for a + # dependency being replaced. + # + # Un-ignore trigger: the azure_storage_blob 1.x migration lands + # (`cargo tree -i azure_core@0.21` returns no rows). + "RUSTSEC-2026-0275", + # quick-xml 0.31.0 — transitive via azure_core 0.21.0 (unofficial SDK, # now archived). Our direct dep is already on 0.41.0; the 0.31 copy is # only reachable through the azure_storage_blobs chain, which parses @@ -40,6 +91,38 @@ ignore = [ "RUSTSEC-2026-0195", "RUSTSEC-2026-0194", + # azure_core 0.21.0 — "Legacy azure_core writes the authorization + # header value to logs" (RUSTSEC-2026-0275, 6.5 medium). Same + # unofficial archived SDK, same absent upgrade path as the other + # 0.21.0-chain advisories above: the advisory's "upgrade to + # >=0.22.0" applies to the official azure_core crate line, not to + # the archived 0.21.0 we're pinned on via the unofficial + # azure_storage_blobs SDK. Real fix is the official azure_core 1.0 / + # azure_storage_blob 1.0 SDK migration tracked separately (memory + # project_azure_sdk_migration_pending) — blocked upstream by the + # 1.0 SDK dropping shared-key auth. + # + # Exposure in this codebase is narrow. The advisory covers the + # HTTP client emitting the `Authorization` header value into log + # records; for our Azure backend usage that header value is + # `SharedKey :` — the shared key itself + # never appears, only a per-request HMAC signature bound to the + # request's `x-ms-date` and unusable outside the ~15 min clock-skew + # window. Reaching the log path further requires (a) an Azure + # backend actually being configured (S3 and local are the + # alternatives) and (b) the tracing subscriber emitting DEBUG + # records for the `azure_core` target — production defaults are + # INFO. Under both conditions the worst-case leak is replay of + # individual object operations within the skew window by an + # attacker who already has production log read access; the shared + # key cannot be derived. + # + # Un-ignore trigger: the official azure_core 1.0 migration lands — + # at which point this entry and the other azure_core 0.21.0-chain + # entries above (RUSTSEC-2026-0097, -2024-0384, -2026-0195, + # -2026-0194) all go away together. + "RUSTSEC-2026-0275", + # wasmtime 43.0.2 — "Stores can mix up type indices between engines" # (GHSA-hgjw-h833-99q9). Transitive via extism 1.30.0 (latest published; # extism `main` still pins wasmtime 43, no upgrade path). The advisory @@ -58,6 +141,35 @@ ignore = [ # with its own Store (see infrastructure/services/plugins/runtime.rs). "RUSTSEC-2026-0222", + # wasmtime 43.0.2 — "Filesystem sandbox escape when paths or symlinks + # contain trailing slashes" (RUSTSEC-2026-0269, 8.8 high). Same crate, + # same chain and same absent upgrade path as RUSTSEC-2026-0222 above: + # extism 1.30.0 is the latest published and pins wasmtime 43, while the + # advisory's fixed releases are >=24.0.13 <25, >=36.0.14 <37, + # >=46.0.3 <47, >=47.0.4 — none in the 43.x line, so there is no + # version satisfying extism's requirement that carries the fix. + # + # NOT REACHABLE, and for a stronger reason than the build-feature + # gating: this is a WASI filesystem sandbox escape, and OxiCloud's + # plugin runtime gives plugins no filesystem to escape from. + # `plugins/runtime.rs::compile` builds every plugin with + # `.with_wasi(false)` and declares no `allowed_paths`, so there are no + # preopened directories — the escape needs one to traverse out of. + # `.disallow_all_hosts()` removes outbound network on the same path. + # + # The build-level gating from the entry above still applies on top: + # `plugins` is opt-in and absent from `default`, so the CI release + # binary does not link wasmtime; runtime activation additionally needs + # OXICLOUD_ENABLE_PLUGINS=true; and plugin binaries are admin-supplied, + # not attacker input. + # + # Un-ignore trigger: EITHER extism releases a version on wasmtime + # >=46.0.3 (check with `cargo tree -i wasmtime --features plugins`), + # OR `plugins/runtime.rs` gains `allowed_paths` / `with_wasi(true)` — + # at which point this stops being unreachable and blocks release + # rather than being ignored. + "RUSTSEC-2026-0269", + # astral-tokio-tar 0.5.6 — tar extraction advisories, transitive via # testcontainers → testcontainers-modules, a DEV-dependency used only by # the `--cfg integration_tests` harness to spin up throwaway Postgres diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b880195c..7a19b1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,21 @@ jobs: with: components: clippy - uses: Swatinem/rust-cache@v2 + + # `--all-features` includes `bundled-assets`, whose build.rs guard + # requires `static-dist/index.html` at compile time (rust-embed + # scans the folder). Build the SPA first so the lint pass covers + # the embed code paths without needing to enumerate features + # around it. ~90 s once, cached by npm-cache on repeats. + - uses: actions/setup-node@v4 + with: + node-version: 26.3.0 + cache: npm + cache-dependency-path: frontend/package-lock.json + - name: Build SPA (needed for --all-features / bundled-assets) + working-directory: frontend + run: npm ci && npm run build + - run: cargo clippy --all-targets --all-features -- -D warnings # Mirrors the `wasm-check` justfile recipe. The wasm crate is a @@ -306,6 +321,18 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + # `--all-features` enables `bundled-assets`, whose build.rs guard + # requires `static-dist/index.html` at compile time. Build the SPA + # first so tests can compile the embed code paths. ~90 s, cached. + - uses: actions/setup-node@v4 + with: + node-version: 26.3.0 + cache: npm + cache-dependency-path: frontend/package-lock.json + - name: Build SPA (needed for --all-features / bundled-assets) + working-directory: frontend + run: npm ci && npm run build + - name: Initialize test database # Applies every migration + seeds the integration-test admin row. # Same script used by `just test-integration` locally. @@ -460,6 +487,79 @@ jobs: path: tests/api/storage/ retention-days: 7 + bundled-binary-test: + # `--features bundled-assets` end-to-end integration test. + # + # Builds oxicloud with the SPA baked in via rust-embed, boots it + # against a nonexistent OXICLOUD_STATIC_PATH so the embed path is + # forced, and asserts SPA + locales + immutable-cache headers + + # CSP all serve correctly from the embedded corpus. Guards against + # three failure classes that don't surface in filesystem-served CI: + # + # 1. rust-embed configuration (glob patterns silently producing a + # 0-file embed — hit 2026-08-28). + # 2. Debug-vs-release drift (rust-embed's dynamic-read mode in + # debug builds masks embed bugs; `debug-embed` feature bakes + # files in for both profiles). + # 3. Axum `Path` extractor on fallback routes returning 500 (the + # `serve_root` handler needs `Request` extraction — hit 2026-08-28). + # + # See tests/bundled-binary/run.sh + docs/plan/bundled-binary.md § 2. + # + # Doesn't reuse the `build` job's artifact because that binary is + # compiled with `--features plugins`, not `--features bundled-assets` + # — different feature set = different target. `Swatinem/rust-cache` + # still shares dependency compilation between the two jobs. + name: Bundled-assets binary — embed + SPA-serve integration + needs: changes + if: | + github.event_name == 'pull_request' && + (needs.changes.outputs.backend == 'true' || needs.changes.outputs.frontend == 'true') + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + # Same disk-hygiene pattern as the `build` job — cargo release + # link + full node_modules install would otherwise squeeze the + # runner disk budget under peak concurrency. + - uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-node@v4 + with: + node-version: 26.3.0 + cache: npm + cache-dependency-path: frontend/package-lock.json + + # Build the SPA up front so the test script can run with + # `SKIP_FRONTEND_BUILD=1` — makes the test-runner log clean + # (no duplicated npm ci noise) and puts the SPA build's cost + # in its own step for CI-side timing visibility. + - name: Build SPA (Vite → static-dist/) + working-directory: frontend + run: npm ci && npm run build + + - name: Run bundled-binary integration test + run: bash tests/bundled-binary/run.sh + env: + SKIP_FRONTEND_BUILD: "1" + + # Preserve the server log even on failure so a red run doesn't + # require re-running locally to see what happened at boot. + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: bundled-binary-server-log + path: tests/bundled-binary/server.log + retention-days: 7 + litmus: name: WebDAV RFC 4918 — litmus (59/59) needs: build diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1bd6e185..7e38a8a3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,7 +1,30 @@ -name: "Docker Hub & GHCR Release" +name: "Docker Publish (release, main, dry-run)" + +# Per-run title shown in the Actions tab list — makes it obvious at +# a glance which channel a given run served and (for dispatched +# runs) whether it was a dry-run. Without this, GitHub falls back +# to the commit subject, which is uninformative when multiple +# workflows fire on the same commit. +# +# Falls back to `github.ref_name` for push / release events (which +# don't carry `inputs.*`), and stitches "[DRY-RUN]" onto the +# dispatched cases where `inputs.dry_run` is checked. +run-name: >- + Docker Publish + ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run && '[DRY-RUN]' || '' }} + — ${{ github.event.inputs.version || github.ref_name }} on: + # Every merge to `main` republishes the mutable `:main` tag so users + # tracking the tip of development can pull the freshest image + # without waiting for a release. `:latest` is DELIBERATELY not + # touched on this trigger — it stays pointed at the last released + # version. Provenance for a specific `:main` pull is queryable via + # `docker inspect` (org.opencontainers.image.revision label carries + # the SHA). push: + branches: + - "main" tags: - "v*" release: @@ -12,14 +35,44 @@ on: version: description: 'Version tag to publish (e.g. v0.5.3)' required: true + dry_run: + description: 'Dry run — build only, skip push + verify. Prints the tag set that WOULD be pushed. Use to smoke-test workflow edits without touching the registry.' + required: false + type: boolean + default: false env: + # Docker Hub image name is fixed to the canonical namespace — forks + # that opt in to publishing typically also override this with their + # own DockerHub account name (see the fork guide in + # docs/plan/docker-publish.md, if/when documented). REGISTRY_IMAGE: diocrafts/oxicloud - GHCR_REGISTRY_IMAGE: ghcr.io/atalayalabs/oxicloud + # GHCR image name follows the repo owner — canonical repo publishes + # to `ghcr.io/atalayalabs/oxicloud`; a fork opting in via + # `vars.ENABLE_DOCKER_PUBLISH=true` publishes to its own owner's + # namespace with zero config edits. + GHCR_REGISTRY_IMAGE: ghcr.io/${{ github.repository_owner }}/oxicloud + +# Cancel superseded `:main` builds if commits land in quick succession +# — only the newest one matters, and having two racing builds pushing +# to the same mutable tag is a coin-toss on which one wins. Release-tag +# and manual-dispatch builds never cancel: each release is unique and +# irreversible; every one must publish. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} jobs: - # Run tests before publishing + # Run tests before publishing. + # + # SKIPPED on push-to-`main`: the same test matrix already ran on the + # PR that produced this commit (branch protection ensures PRs pass + # CI before merge). Re-running here would double the CI cost per + # merge for zero signal. Release-tag / manual-dispatch builds still + # test — they're explicit "shipping this" moments where + # belt-and-suspenders matters. test: + if: github.event_name != 'push' || !startsWith(github.ref, 'refs/heads/') name: Pre-publish Tests runs-on: ubuntu-latest services: @@ -60,30 +113,64 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 360 needs: test + # Publish gate — TWO conditions must hold: + # + # 1. The `test` job succeeded (or was skipped for push-to-main — + # branch protection ensures PR CI already ran; see the test + # job's `if:`). `always()` unblocks the `needs:` when test is + # skipped; the result check still blocks on real failures. + # + # 2. Publishing is enabled for THIS repo. Canonical + # `AtalayaLabs/OxiCloud` always publishes. Forks stay quiet by + # default (no accidental GHCR packages / wasted CI minutes when + # someone forks just to read code); a fork owner who wants to + # test-publish sets `ENABLE_DOCKER_PUBLISH=true` under + # Settings → Secrets and variables → Actions → Variables. + if: | + always() && + (needs.test.result == 'success' || needs.test.result == 'skipped') && + (github.repository == 'AtalayaLabs/OxiCloud' || vars.ENABLE_DOCKER_PUBLISH == 'true') permissions: contents: read packages: write + # Job-level env — `secrets` context is legal here but NOT in + # step-level `if:` conditions. Precomputing the "is DH configured" + # signal as an env var lets downstream steps gate cleanly via + # `env.HAS_DOCKERHUB_TOKEN == 'true'` — see the DockerHub login + # step below. + env: + HAS_DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN != '' }} steps: - name: Checkout uses: actions/checkout@v4 with: - # Build the exact tag behind the published release or manual dispatch. + # Build the exact tag behind the published release or manual + # dispatch. On push-to-main, `github.ref` resolves to + # `refs/heads/main` and this checks out the freshly-merged + # commit — exactly what we want to publish as `:main`. ref: ${{ github.event.inputs.version || github.event.release.tag_name || github.ref }} - - name: Set version tag - id: version - run: | - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - VERSION="${{ github.event.inputs.version }}" - elif [ "${{ github.event_name }}" == "release" ]; then - VERSION="${{ github.event.release.tag_name }}" - else - VERSION="${GITHUB_REF#refs/tags/}" - fi - # Strip leading 'v' if present for Docker tag - VERSION="${VERSION#v}" - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Resolve channel + version + tags + id: meta + # Delegates to `scripts/compute-docker-tags.sh` — logic is + # unit-tested via `scripts/test-docker-publish-tags.sh` so + # any change to the tag policy gets caught before hitting a + # runner. The script emits `version`, `channel`, and `tags` + # to $GITHUB_OUTPUT (for later `steps.meta.outputs.*`), + # plus `VERSION` / `CHANNEL` / `SKIP_DOCKERHUB` to + # $GITHUB_ENV (for later steps that read env directly), plus + # a human-readable trailer to stdout for the run log — + # useful in dry-run mode where the tag set is the deliverable. + env: + EVENT_NAME: ${{ github.event_name }} + GITHUB_REF: ${{ github.ref }} + DISPATCH_VERSION: ${{ github.event.inputs.version }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + # Empty DOCKERHUB_TOKEN → skip DH tags cleanly (forks that + # opted in via ENABLE_DOCKER_PUBLISH but didn't set up + # DockerHub still get their GHCR image published). + SKIP_DOCKERHUB: ${{ secrets.DOCKERHUB_TOKEN == '' && 'true' || 'false' }} + run: bash "$GITHUB_WORKSPACE/scripts/compute-docker-tags.sh" - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -92,6 +179,19 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to DockerHub + # Skipped cleanly when DOCKERHUB_TOKEN isn't configured on + # this repo — the meta step's SKIP_DOCKERHUB env drops DH + # tags from the push set in that case, so we don't need + # DH auth at all. Canonical repo always has the secret and + # always publishes; forks are opt-in via ENABLE_DOCKER_PUBLISH + # AND can further opt in / out of DH separately by + # adding / omitting DOCKERHUB_TOKEN. + # + # `secrets` context is not available in step-level `if:` + # conditions — we read it via the job-level env var + # `HAS_DOCKERHUB_TOKEN` computed above (which CAN reference + # secrets since it lives in `env:`, not `if:`). + if: env.HAS_DOCKERHUB_TOKEN == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -109,12 +209,17 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64 - push: true - tags: | - ${{ env.REGISTRY_IMAGE }}:${{ env.VERSION }} - ${{ env.REGISTRY_IMAGE }}:latest - ${{ env.GHCR_REGISTRY_IMAGE }}:${{ env.VERSION }} - ${{ env.GHCR_REGISTRY_IMAGE }}:latest + # `push` flips to `false` for a dry-run — the multi-arch + # build still runs (catches Dockerfile regressions), but + # nothing hits the registry. Only reachable via + # `workflow_dispatch` with `dry_run: true`. Real push + # events (release, branch push) always publish. + push: ${{ github.event.inputs.dry_run != 'true' }} + # Tag set computed in the meta step above — release channel + # publishes `:` + `:latest`; main channel publishes + # just `:main`. Emitted to the build log either way so the + # dry-run mode surfaces "what would ship" in plain sight. + tags: ${{ steps.meta.outputs.tags }} cache-from: type=gha cache-to: type=gha,mode=max # GitHub Actions env piped through so build.rs stamps @@ -129,7 +234,36 @@ jobs: GITHUB_HEAD_REF=${{ github.head_ref }} - name: Verify published image + # Skipped on dry-run — nothing was pushed to pull back. + # Verifies GHCR unconditionally (always pushed), then adds a + # Docker Hub pull when the DH branch actually ran. Simpler + # than a matrix — the two registries share the same content + # (same multi-arch manifest), so one pull confirms the build + # + push worked; the other is just a "did we auth to both" + # sanity check. + if: github.event.inputs.dry_run != 'true' run: | - docker pull ${{ env.REGISTRY_IMAGE }}:${{ env.VERSION }} - docker image inspect ${{ env.REGISTRY_IMAGE }}:${{ env.VERSION }} - echo "✅ Image ${{ env.REGISTRY_IMAGE }}:${{ env.VERSION }} published successfully" + echo "─── Verify GHCR ───" + docker pull ${{ env.GHCR_REGISTRY_IMAGE }}:${{ env.VERSION }} + docker image inspect ${{ env.GHCR_REGISTRY_IMAGE }}:${{ env.VERSION }} > /dev/null + echo "✅ ${{ env.GHCR_REGISTRY_IMAGE }}:${{ env.VERSION }} published" + if [ "${{ env.SKIP_DOCKERHUB }}" != "true" ]; then + echo "─── Verify Docker Hub ───" + docker pull ${{ env.REGISTRY_IMAGE }}:${{ env.VERSION }} + docker image inspect ${{ env.REGISTRY_IMAGE }}:${{ env.VERSION }} > /dev/null + echo "✅ ${{ env.REGISTRY_IMAGE }}:${{ env.VERSION }} published" + else + echo "ℹ️ Skipped Docker Hub verification (DOCKERHUB_TOKEN not set on this repo)" + fi + + - name: Dry-run summary + # Only surfaces in dry-run mode. Mirrors the "Verify" step's + # role — gives the operator running the dry-run a clear + # closing message with the exact tag set the workflow would + # have pushed. The meta step already logged it, this step + # just makes it prominent at the bottom of the run. + if: github.event.inputs.dry_run == 'true' + run: | + echo "🔍 DRY RUN — image built + tagged but NOT pushed." + echo "Would have published:" + echo "${{ steps.meta.outputs.tags }}" | sed 's/^/ /' diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml new file mode 100644 index 00000000..a0eb3072 --- /dev/null +++ b/.github/workflows/release-binaries.yml @@ -0,0 +1,288 @@ +name: "Release Binaries (musl-linux + macOS)" + +# Per-run title shown in the Actions tab — makes it obvious at a +# glance which tag is being packaged and whether a manual run is a +# dry-run (build tarballs into workflow artifacts, DON'T attach to +# any GitHub Release). +run-name: >- + Release Binaries + ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run && '[DRY-RUN]' || '' }} + — ${{ github.event.inputs.version || github.ref_name }} + +# TRIGGERS — deliberately narrow. This workflow builds 4 platform +# binaries (~15-25 min wall-clock, matrix of native runners) and +# attaches them to a GitHub Release. Running on every push to main +# would be gratuitous CI cost + noise — the point is to package +# releases, not to sanity-check the tip. The bundled-binary +# integration test in ci.yml already covers "does the embed still +# work" on every PR. +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + version: + description: 'Existing tag to package (e.g. v0.9.0). Must exist on origin.' + required: true + dry_run: + description: 'Dry run — build + upload tarballs as workflow artifacts, skip attaching to a Release. Use to smoke-test workflow edits without publishing.' + required: false + type: boolean + default: false + +# Concurrency key includes the tag ref so different tags don't cancel +# each other; `cancel-in-progress: false` because tag builds are +# unique + immutable — a superseded release build has nothing to cancel. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: write + +jobs: + # ── Stage 1: Build the SPA once, share across all platforms ───────── + # + # The SvelteKit build is arch-independent so a single ubuntu runner + # produces static-dist/ for every downstream binary-build matrix + # entry — saves ~2 min × 4 = 8 min vs building it per platform. + frontend-build: + name: Build SPA (Vite → static-dist/) + # Publish gate — same fork-friendly pattern as docker-publish.yml. + # Canonical repo always builds; forks stay quiet unless the fork + # owner opts in via `vars.ENABLE_BINARY_RELEASE=true` under Settings + # → Secrets and variables → Actions → Variables. + if: | + github.repository == 'AtalayaLabs/OxiCloud' || + vars.ENABLE_BINARY_RELEASE == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # Build the exact tag being packaged. `github.ref` is + # refs/tags/vX.Y.Z on push, refs/heads/... on dispatch (we + # override via `inputs.version` in that case). + ref: ${{ github.event.inputs.version || github.ref }} + + - uses: actions/setup-node@v4 + with: + node-version: 26.3.0 + cache: npm + cache-dependency-path: frontend/package-lock.json + + - name: Build SPA + working-directory: frontend + run: npm ci && npm run build + + - uses: actions/upload-artifact@v4 + with: + name: static-dist + # Repo-root output (SvelteKit adapter-static's `pages: + # '../static-dist'`). Downstream jobs restore it to the same + # location so rust-embed's `#[folder = "static-dist/"]` + # resolves without any path juggling. + path: static-dist/ + retention-days: 1 + + # ── Stage 2: Build one binary per target ──────────────────────────── + # + # 3-way matrix — 2 musl-linux (native amd64 + arm64) + macOS Apple + # Silicon. Windows and Intel macOS are deferred: + # + # * Intel macOS (`x86_64-apple-darwin` / macos-13 runner) — dropped + # 2026-08-29. Apple is phasing out Intel Macs from GitHub's hosted + # runners; the `macos-13` tier is scheduled for deprecation and + # queues stretched past 1 h during v0.9.0-rc1 build. Intel Mac + # users have three fallbacks: (1) `cargo install oxicloud + # --locked --features bundled-assets` from source, (2) `docker + # pull --platform linux/amd64 ghcr.io/atalayalabs/oxicloud`, + # (3) any of the two Linux musl tarballs via a Linux VM. The + # Intel-Mac install base is small and shrinking (Apple Silicon + # >90% of new sales) so first-class shipping isn't worth the + # CI-availability tax. + # * Windows — separate work when demand appears. + # + # All targets run natively on GitHub-hosted runners with the host's + # glibc + rustup, then cross-compile to their target triple via + # `rustup target add`. The musl-linux targets install `musl-tools` + # (which provides `musl-gcc`) so aws-lc-sys and friends can link + # against musl. macOS runners already have the apple-* triples + # pre-installed. + # + # History: an earlier draft ran Linux builds INSIDE the + # `rust:1.96-alpine3.24` container the Dockerfile uses — matched + # Docker image byte-for-byte. Broke on `ubuntu-22.04-arm`: GitHub + # Actions JS-based actions (`actions/checkout`, artifact steps, + # setup-node) can't run inside Alpine on ARM64 — the Node.js binary + # they ship depends on glibc, and the x64-Alpine workaround doesn't + # extend to arm64. Native ubuntu + musl-tools sidesteps the whole + # thing and produces the same output (both are `cargo build + # --target x86_64-unknown-linux-musl` / `aarch64-...-musl`). + binary-build: + name: Build ${{ matrix.triple }} + needs: frontend-build + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + # `fail-fast: false` — one platform's compile failure shouldn't + # cancel the other three. Partial releases are better than none. + fail-fast: false + matrix: + include: + - triple: x86_64-unknown-linux-musl + runner: ubuntu-22.04 + rustflags: "-C target-cpu=x86-64-v2" + - triple: aarch64-unknown-linux-musl + runner: ubuntu-22.04-arm + # ARMv8-A baseline — covers Pi 4/5, Graviton, every 64-bit + # ARM Linux server. `generic` is rustc's neutral baseline. + rustflags: "-C target-cpu=generic" + - triple: aarch64-apple-darwin + runner: macos-latest + rustflags: "-C target-cpu=apple-m1" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.version || github.ref }} + + # musl-tools ships `musl-gcc` — the C-compiler wrapper that rustc + # invokes for -musl targets. Without it, `cargo build --target + # aarch64-unknown-linux-musl` fails with "linker `musl-gcc` not + # found" on any C-linked dep (aws-lc-sys, ring, sqlx's native + # backend when enabled). + - name: Install musl-tools (Linux only) + if: contains(matrix.triple, '-linux-musl') + run: sudo apt-get update && sudo apt-get install -y musl-tools + + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.triple }} + + - uses: Swatinem/rust-cache@v2 + with: + # Key by triple so the 4 targets don't share caches + # (different feature set + different target triple = different + # compiled artefacts). + key: ${{ matrix.triple }} + + - uses: actions/download-artifact@v4 + with: + name: static-dist + path: static-dist/ + + # `--features bundled-assets` bakes static-dist/ into the binary + # via rust-embed. `--bin oxicloud` — the single binary the merge + # (Deliverable 1b) consolidated everything into. + - name: Build binary + env: + # Per-triple CPU baseline — release binaries target the widest + # realistic install base for their arch. See + # docs/plan/bundled-binary.md § 3. + RUSTFLAGS: ${{ matrix.rustflags }} + # Git metadata injection — build.rs reads these env vars to + # stamp GIT_HASH / GIT_BRANCH into the binary. Without them + # `oxicloud --version` reports "unknown". + GITHUB_SHA: ${{ github.sha }} + GITHUB_REF_NAME: ${{ github.ref_name }} + run: | + cargo build --release --features bundled-assets --bin oxicloud --target ${{ matrix.triple }} + + # Assemble the tarball layout documented in + # docs/plan/bundled-binary.md § 4: oxicloud + example.env + + # LICENSE + README-install.md, rooted under a per-version-per- + # triple directory so `tar xzf` lands cleanly. + - name: Package tarball + run: | + set -euo pipefail + # Version = tag stripped of leading `v` (workflow_dispatch) + # or ref_name stripped (push tag). Falls back to ref_name + # verbatim if neither strip matches. + RAW_REF="${{ github.event.inputs.version || github.ref_name }}" + VERSION="${RAW_REF#v}" + DIST="oxicloud-${VERSION}-${{ matrix.triple }}" + mkdir -p "dist/${DIST}" + cp "target/${{ matrix.triple }}/release/oxicloud" "dist/${DIST}/oxicloud" + cp example.env "dist/${DIST}/example.env" + cp LICENSE "dist/${DIST}/LICENSE" + # README-install.md may not exist yet in early releases — + # ship a stub that points at the docs site so users have + # something in the tarball. Deliverable 6 replaces it with + # a proper install guide. + if [ -f docs/install/binary.md ]; then + cp docs/install/binary.md "dist/${DIST}/README-install.md" + else + cat > "dist/${DIST}/README-install.md" <<'MD' + # OxiCloud — Installation + + Full documentation: https://github.com/AtalayaLabs/OxiCloud/tree/main/docs + + Quickstart: + 1. Set DATABASE_URL to a PostgreSQL 13+ instance + (with pg_trgm + ltree extensions). + 2. Copy example.env → .env, edit as needed. + 3. Run ./oxicloud. + + Optional: install ffmpeg for server-side video thumbnails + (or set OXICLOUD_ENABLE_VIDEO_THUMBNAILS=false to disable). + MD + fi + # Deterministic tar (owner/group/mtime pinned) so re-running + # the build produces byte-identical archives — helps with + # reproducible-build audits and cheap hash verification. + tar --owner=0 --group=0 -czf "dist/${DIST}.tar.gz" -C dist "${DIST}" + ls -la "dist/${DIST}.tar.gz" + + - uses: actions/upload-artifact@v4 + with: + name: tarball-${{ matrix.triple }} + path: dist/*.tar.gz + retention-days: 1 + + # ── Stage 3: Attach all tarballs + SHA256SUMS to the Release ──────── + # + # `dry_run: true` (workflow_dispatch only) skips this job — the + # binary tarballs stay as workflow artifacts (accessible from the + # run page for 1 day) but nothing lands on any Release. + release: + name: Attach tarballs to GitHub Release + needs: binary-build + if: | + needs.binary-build.result == 'success' && + (github.event_name != 'workflow_dispatch' || github.event.inputs.dry_run != 'true') + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + pattern: tarball-* + path: dist/ + merge-multiple: true + + - name: Compute SHA256SUMS + run: | + set -euo pipefail + cd dist + # Sort output for stable ordering across re-runs — the file + # doubles as a manifest an operator can `diff` between two + # release runs to prove they're identical. + sha256sum *.tar.gz | sort > SHA256SUMS + cat SHA256SUMS + + # softprops/action-gh-release@v2 semantics: + # - If the Release for this tag EXISTS (created by release.yml + # which runs in parallel on the same tag push), attaches the + # files to it. + # - If it doesn't yet exist (race — release.yml still running), + # creates a bare Release which release.yml then fills in with + # notes when it finishes. + # Benign either way; see docs/plan/bundled-binary.md § 5 + # "Parallel-fire behaviour on tag push". + - name: Attach to Release + uses: softprops/action-gh-release@v2 + with: + files: | + dist/*.tar.gz + dist/SHA256SUMS + fail_on_unmatched_files: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 2ee0cc0f..96c38234 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "adobe-cmap-parser" version = "0.4.1" @@ -601,7 +607,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.13", + "h2 0.4.16", "http 0.2.12", "http 1.4.0", "http-body 0.4.6", @@ -1172,6 +1178,16 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -1962,6 +1978,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + [[package]] name = "dashmap" version = "6.2.1" @@ -2884,6 +2906,19 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "group" version = "0.13.0" @@ -2916,9 +2951,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -3197,7 +3232,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.13", + "h2 0.4.16", "http 1.4.0", "http-body 1.0.1", "httparse", @@ -3507,6 +3542,39 @@ dependencies = [ "quick-error", ] +[[package]] +name = "include-flate" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f173716febb1ad596c16ea5637b5f1790ea32de8e627493ff82bc73b0876ce" +dependencies = [ + "include-flate-codegen", + "include-flate-compress", +] + +[[package]] +name = "include-flate-codegen" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a7875b62a72ad3f3203cdd8950d4cf9947db036030b974b8b37ceae90c8d8c0" +dependencies = [ + "include-flate-compress", + "proc-macro-error3", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "include-flate-compress" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fbb9c5ccb9a5b67b4afa2974c27e5507ea1bf6d22828cef418e4dfaeca51dd" +dependencies = [ + "libflate", + "zstd", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -3802,6 +3870,30 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libflate" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c" +dependencies = [ + "adler32", + "crc32fast", + "dary_heap", + "libflate_lz77", + "no_std_io2", +] + +[[package]] +name = "libflate_lz77" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" +dependencies = [ + "hashbrown 0.16.1", + "no_std_io2", + "rle-decode-fast", +] + [[package]] name = "libloading" version = "0.9.0" @@ -4278,6 +4370,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + [[package]] name = "nom" version = "7.1.3" @@ -4532,7 +4633,7 @@ dependencies = [ [[package]] name = "oxicloud" -version = "0.8.7" +version = "0.8.9" dependencies = [ "accept-language", "aes-gcm", @@ -4597,6 +4698,7 @@ dependencies = [ "rand_core 0.6.4", "rayon", "reqwest", + "rust-embed", "serde", "serde_json", "sha2 0.11.0", @@ -5006,6 +5108,28 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error-attr3" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0084e6206a967a2dad822180626b2f6b07a3b379325e8f1ec0438e33a469ba7" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error3" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cf066225f2373bc711684792b69bdeac0356019b007e721090c24d92d5d5a50" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", + "syn 3.0.2", +] + [[package]] name = "proc-macro-utils" version = "0.10.0" @@ -5558,6 +5682,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "rmp" version = "0.8.15" @@ -5597,6 +5727,44 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-embed" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e7760e252aaba7b09f4be00e36476cf585bdb68a53552ac954cdf504ab4bc9" +dependencies = [ + "include-flate", + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bcfc4d6f53af43755f7a723e4b6b8794fcce052a178dd8c6c1dadc5f5343097" +dependencies = [ + "mime_guess", + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.117", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ffa149f6aa81b58a5b3011d01a857c4ed12c7a732d2c51947a4c7c692185f0" +dependencies = [ + "globset", + "include-flate", + "sha2 0.11.0", + "walkdir", +] + [[package]] name = "rust-stemmers" version = "1.2.0" @@ -7016,7 +7184,7 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.13", + "h2 0.4.16", "http 1.4.0", "http-body 1.0.1", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index 346ba04a..16cbb6c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,16 +1,34 @@ [package] name = "oxicloud" -version = "0.8.7" +version = "0.8.9" edition = "2024" default-run = "oxicloud" +# `cargo binstall oxicloud` — fetches the prebuilt release tarball for the +# host triple from GitHub Releases (attached by +# `.github/workflows/release-binaries.yml`) instead of compiling from +# source. Templates match the tarball naming +# `oxicloud--.tar.gz` produced by that workflow. +# +# `pkg-fmt = "tgz"` — otherwise binstall guesses from the URL extension; +# being explicit lets `cargo binstall oxicloud` succeed on Windows too +# (where the URL string parsing differs). +# +# Once the first tagged release lands on GitHub, this becomes a one-line +# install for anyone with the Rust toolchain who prefers not to build +# from source and doesn't want Docker either. +[package.metadata.binstall] +pkg-url = "{ repo }/releases/download/v{ version }/oxicloud-{ version }-{ target }.tar.gz" +pkg-fmt = "tgz" +bin-dir = "oxicloud-{ version }-{ target }/{ bin }{ binary-ext }" + [dependencies] mimalloc = { version = "0.1.52", default-features = false } axum = { version = "0.8.9", features = ["multipart", "http1", "http2", "tokio", "macros"] } # "process" was previously enabled implicitly through aws-config's feature # unification; ffmpeg_video_frame_service needs it, so declare it ourselves. -tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "io-util", "net", "time", "sync", "fs", "process"] } +tokio = { version = "1.52.3", features = ["rt-multi-thread", "macros", "io-util", "net", "time", "sync", "fs", "process", "signal"] } tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] } tokio-stream = { version = "0.1.18", features = ["fs", "sync"] } bytes = "1.11.1" @@ -47,6 +65,25 @@ futures = "0.3.32" async-stream = "0.3.6" async-trait = "0.1.89" mime_guess = "2.0.5" +# `rust-embed` — compile-time asset embedding for the `bundled-assets` feature. +# Optional so default builds never pull it in. +# +# Features: +# * `compression` — deflate-compress each embedded file at compile time; the +# handler decompresses lazily on first read (cached per-file in a +# `OnceCell`). Halves the on-disk contribution to the final binary. +# * `include-exclude` — enables the `#[include]` / `#[exclude]` glob +# attributes on the derive. +# * `debug-embed` — CRITICAL. Without this, debug builds read files from +# disk at runtime (dynamic mode) instead of compiling them in. The +# runtime read is fragile (`CARGO_MANIFEST_DIR` resolution + working +# directory dependency) and returned `total=0` for us on 2026-08-28. +# With `debug-embed`, both `cargo build` and `cargo build --release` +# produce a truly self-contained binary — the only sensible default +# for the `bundled-assets` feature. +# +# See src/interfaces/web/embedded.rs for the actual embed struct + handlers. +rust-embed = { version = "8", features = ["compression", "include-exclude", "debug-embed"], optional = true } uuid = { version = "1.23.3", features = ["v4", "v7", "serde"] } thiserror = "2.0.18" arc-swap = "1.9" @@ -168,6 +205,31 @@ faces-onnx = ["dep:ort", "dep:ndarray"] # `examples/` can measure them. Off by default — adds nothing to prod builds. # Run with: `cargo bench --features bench` / `cargo run --release --features bench --example bench_thumbnails_mem`. bench = [] +# Empty marker feature that gates the `generate-openapi` binary out of the +# default release build set. `just openapi` flips it when the SPA needs a +# regenerated openapi.json; end-user release builds never do. Kept separate +# from `test_utils` for the same reason `load_seed_bin` is — enabling +# `dev_tools` on the CLI must not perturb the oxicloud dependency graph. +dev_tools = [] +# Bake the SvelteKit build output (`static-dist/` at repo root) into the +# binary at compile time via `rust-embed`. Opt-in and off by default — the +# regular `cargo build --release` still produces a filesystem-served +# binary (matching the current Docker image where assets are separate +# layers, and the `just dev` HMR loop where Vite serves live). Release +# tarballs (`docs/plan/bundled-binary.md` § 2) build with this flag to +# ship a single self-contained executable. +# +# Precedence rule preserved: even when this feature is on, if +# OXICLOUD_STATIC_PATH points at an existing directory, that wins over +# the embedded fallback — ops can override embedded assets for locale +# patches or theming without a rebuild. +# +# Build-time invariant: `cargo build --features bundled-assets` requires +# `static-dist/` at the repo root (SvelteKit adapter-static emits there +# — `frontend/svelte.config.js`'s `pages: '../static-dist'`). `build.rs` +# fails fast with a pointer to `(cd frontend && npm run build)` when the +# directory is missing. +bundled-assets = ["dep:rust-embed"] [dev-dependencies] criterion = "0.5" @@ -186,19 +248,12 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(integration_tests)'] } [[bin]] name = "generate-openapi" path = "src/bin/generate-openapi.rs" - -[[bin]] -name = "migrate-nfc-filenames" -path = "src/bin/migrate-nfc-filenames.rs" - -[[bin]] -name = "oxicloud-cli" -path = "src/bin/oxicloud-cli.rs" -# Operator toolbox — subcommand-driven CLI for tasks that don't belong -# in the main server. Currently: `oxicloud-cli opaque {setup,reset}`. -# Ships in the release Dockerfile as the single operator-facing helper -# (replaces the earlier per-task `opaque-setup` bin, which was folded -# into `oxicloud-cli opaque setup`). +# Dev-only: regenerates `resources/gen/openapi.json` from the utoipa +# `#[utoipa::path]` annotations in the API handlers. Gated behind the +# `dev_tools` feature so `cargo build --release --bins` (and the prod +# Dockerfile) skip it entirely — end users have no reason to run it. +# Invoked by `just openapi`, which passes `--features dev_tools`. +required-features = ["dev_tools"] [[bin]] name = "opaque-hurl-helper" @@ -207,7 +262,10 @@ path = "src/bin/opaque-hurl-helper.rs" # handshake against a running server. Invoked from tests/api/run.sh # after opaque_substrate.hurl to cover the parts Hurl can't (OPRF # blinding, AKE nonces are per-attempt-random). Not shipped in the -# release Dockerfile (nothing outside tests/ calls it). +# release Dockerfile (nothing outside tests/ calls it). Gated behind +# `test_utils` so `cargo build --release --bins` skips it; `run.sh` +# enables the feature explicitly when building the helper on demand. +required-features = ["test_utils"] [[bin]] name = "dpop-hurl-helper" @@ -216,7 +274,8 @@ path = "src/bin/dpop-hurl-helper.rs" # the DPoP-Nonce challenge/retry loop, and covers the wire-protocol # scenarios Hurl can't express (per-request fresh jti/iat, replay # detection, malformed proofs, wrong htm/htu/alg/typ). Same -# no-ship-in-release status as opaque-hurl-helper. +# no-ship-in-release status as opaque-hurl-helper — gated identically. +required-features = ["test_utils"] [[bin]] name = "load-seed" @@ -938,11 +997,20 @@ opt-level = 1 debug = "line-tables-only" split-debuginfo = "unpacked" # Incremental compilation caches per-function IR fingerprints so a -# small edit only recompiles what changed. On a single-crate rebuild -# (oxicloud is one crate) the savings are modest — worth < the ~7 GB -# incremental/ cache costs on disk. Rust-analyzer uses `cargo check`, -# which has its own cache, so LSP responsiveness is unaffected. -incremental = false +# small edit only recompiles what changed. The `target/incremental/` +# cache costs ~7 GB per profile, but at the current codebase size a +# full rebuild is ~10 minutes and an incremental single-file edit is +# seconds — the disk is worth it and then some. Rust-analyzer's own +# `cargo check` cache is separate; LSP responsiveness is unaffected +# either way. +# +# History: this was `= false` early on when the crate was small and +# incremental's savings didn't cover the disk cost. Re-enabled +# 2026-08-29 as the full-rebuild time crossed the "feels annoying" +# threshold on typical dev-loop edits. If disk pressure ever spikes, +# `cargo clean -p oxicloud --profile dev` clears the incremental cache +# without wiping compiled deps. +incremental = true [profile.bench] lto = "fat" diff --git a/Dockerfile b/Dockerfile index 0f86c094..39c1f1a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,11 +42,8 @@ COPY build.rs ./ # Create a minimal project to download and cache dependencies RUN mkdir -p src/bin && \ echo 'fn main() { println!("Dummy build for caching dependencies"); }' > src/main.rs && \ - echo 'fn main() {}' > src/bin/generate-openapi.rs && \ - echo 'fn main() {}' > src/bin/migrate-nfc-filenames.rs && \ - echo 'fn main() {}' > src/bin/oxicloud-cli.rs && \ echo 'fn main() {}' > src/bin/opaque-hurl-helper.rs && \ - cargo build --release --bin oxicloud --bin generate-openapi --bin migrate-nfc-filenames --bin oxicloud-cli && \ + cargo build --release --bin oxicloud && \ rm -rf src static-dist target/release/deps/oxicloud* target/release/build/oxicloud-* # ─── Stage 3: Build the application ────────────────────────────────────────── @@ -86,7 +83,7 @@ RUN DATABASE_URL="${DATABASE_URL}" \ GITHUB_SHA="${GITHUB_SHA}" \ GITHUB_REF_NAME="${GITHUB_REF_NAME}" \ GITHUB_HEAD_REF="${GITHUB_HEAD_REF}" \ - cargo build --release --bin oxicloud --bin generate-openapi --bin migrate-nfc-filenames --bin oxicloud-cli + cargo build --release --bin oxicloud # The SPA is built by the Vite frontend stage; bring it in for the runtime copy # below (build.rs has no asset pipeline — it only injects git metadata). COPY --from=frontend /static-dist ./static-dist @@ -126,21 +123,24 @@ RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry,sharin GITHUB_HEAD_REF="${GITHUB_HEAD_REF}" \ cargo build --release && \ mkdir -p /app/bin && \ - cp target/release/oxicloud /app/bin/oxicloud && \ - cp target/release/migrate-nfc-filenames /app/bin/migrate-nfc-filenames && \ - cp target/release/oxicloud-cli /app/bin/oxicloud-cli + cp target/release/oxicloud /app/bin/oxicloud # ─── Stage 3c: Select the builder & normalise the binary path ───────────────── # FROM expands the global ${BUILDER} arg to alias the chosen builder stage # (`builder` for CI/release, `builder-cache` for the e2e image). It then copies -# the two shipped binaries from the builder-specific ${BIN_DIR} into a single -# stable path (/app/release) so the runtime stage's COPYs are independent of -# which builder ran. `static-dist` already lives at /app/static-dist in both +# the shipped binary from the builder-specific ${BIN_DIR} into a single stable +# path (/app/release) so the runtime stage's COPY is independent of which +# builder ran. `static-dist` already lives at /app/static-dist in both # builders, so it needs no normalisation. +# +# Single `oxicloud` binary since v0.9.0 — the operator toolbox +# (`opaque setup`, `migrate nfc-filenames`, …) now lives under +# `oxicloud ` rather than in standalone `oxicloud-cli` / +# `migrate-nfc-filenames` bins. See docs/plan/bundled-binary.md § 1b. FROM ${BUILDER} AS app ARG BIN_DIR RUN mkdir -p /app/release && \ - cp "${BIN_DIR}/oxicloud" "${BIN_DIR}/migrate-nfc-filenames" "${BIN_DIR}/oxicloud-cli" /app/release/ + cp "${BIN_DIR}/oxicloud" /app/release/ # ─── Stage 4: Minimal runtime image ────────────────────────────────────────── FROM alpine:3.24.0 @@ -163,21 +163,18 @@ RUN apk --no-cache upgrade && \ addgroup -g 1001 -S oxicloud && \ adduser -u 1001 -S oxicloud -G oxicloud -# Copy the compiled binary and entrypoint (--chmod avoids extra RUN chmod layers) +# Copy the compiled binary and entrypoint (--chmod avoids extra RUN chmod layers). +# +# Single `oxicloud` binary — since v0.9.0 the operator toolbox lives +# under `oxicloud ` rather than as standalone helper bins: +# +# docker run --rm oxicloud opaque setup # print OPAQUE ServerSetup +# docker exec oxicloud migrate nfc-filenames --dry-run +# # NFC-normalize storage.files.name (pre-June-2026 dbs; safe on new installs) +# +# Bare `oxicloud` (Docker CMD default) still starts the server — backwards +# compat preserved. See docs/plan/bundled-binary.md § 1b. COPY --from=app --chmod=755 /app/release/oxicloud /usr/local/bin/ -# Ship the NFC filename migration binary alongside the server so -# operators can run it inside the container without a separate Rust -# toolchain — `docker exec migrate-nfc-filenames --dry-run` -# to preview, drop `--dry-run` to execute. One-shot tool, safe to -# ship; it only mutates `storage.files` rows whose name ≠ NFC(name). -COPY --from=app --chmod=755 /app/release/migrate-nfc-filenames /usr/local/bin/ -# Ship the OPAQUE server-setup generator alongside the server so operators -# can generate their `OXICLOUD_AUTH_OPAQUE_SERVER_SETUP` value inside the -# container without a separate Rust toolchain: -# docker run --rm oxicloud-cli opaque setup # prints the base64 value -# One-shot, side-effect-free — safe to include; the runtime doesn't -# invoke it, admins do (see docs/config/authentication.md §OPAQUE). -COPY --from=app --chmod=755 /app/release/oxicloud-cli /usr/local/bin/ COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN sed -i 's/\r//' /usr/local/bin/entrypoint.sh && \ chmod 755 /usr/local/bin/entrypoint.sh diff --git a/README.md b/README.md index 0ff14ec5..b5b5450a 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,16 @@ docker compose up -d Open `http://localhost:8086`. +### Prebuilt binary + +Binary releases (Linux musl amd64/arm64, macOS Intel/Apple Silicon) +are attached to every tagged release on GitHub — the whole SPA + all +operator subcommands + migrations bake into a single self-contained +executable. See [`docs/install/binary.md`](docs/install/binary.md) for +the download / verify / systemd walkthrough. + +`cargo binstall oxicloud` works too once a release is out. + ### Run from source Requires Rust 1.93+ and PostgreSQL. diff --git a/build.rs b/build.rs index 282199e0..28ce7298 100644 --- a/build.rs +++ b/build.rs @@ -1,15 +1,58 @@ -//! build.rs — injects git build metadata into the binary. +//! build.rs — injects git build metadata into the binary and, under the +//! optional `bundled-assets` feature, guards the compile-time embed +//! precondition. //! //! Exposes `GIT_HASH` and `GIT_BRANCH` (consumed via `env!()` in `main.rs`). -//! There is no Rust-side asset pipeline: the frontend is built by Vite into -//! `static-dist/` and served directly by the web layer (`interfaces::web`). +//! The frontend is built by Vite into `static-dist/` at the repo root and +//! served directly by the web layer (`interfaces::web`); when +//! `bundled-assets` is on, `src/interfaces/web/embedded.rs` bakes that +//! directory into the binary at compile time via `rust-embed`. use std::env; +use std::path::Path; use std::process::Command; fn main() { println!("cargo:rerun-if-changed=build.rs"); git_status(); + bundled_assets_guard(); +} + +// ═══════════════════════════════════════════════════════════════════════════════ +// Bundled-assets precondition guard +// +// When `--features bundled-assets` is on, `rust-embed`'s `#[folder = "static-dist/"]` +// scans that directory at compile time and errors with a not-very-helpful +// "No such file or directory" if it's missing. Users hit this first when they +// try `cargo build --release --features bundled-assets` before running the +// frontend build — we intercept it here with a clear, actionable message. +// +// Also emits `cargo:rerun-if-changed=static-dist/` so a fresh frontend build +// re-triggers the embed step without needing `cargo clean` — matches what a +// dev on the bundled feature would expect after `just fe-build`. +// ═══════════════════════════════════════════════════════════════════════════════ +fn bundled_assets_guard() { + if env::var("CARGO_FEATURE_BUNDLED_ASSETS").is_err() { + return; + } + println!("cargo:rerun-if-changed=static-dist"); + + let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); + let dist = Path::new(&manifest_dir).join("static-dist"); + let index = dist.join("index.html"); + if !index.exists() { + // `cargo:warning=` prefixes surface these in the terminal even + // when cargo's default output is quiet; the panic below turns + // them into a compile-time error so the missing prerequisite + // can't slip past a distracted dev. + println!("cargo:warning=`bundled-assets` feature requires static-dist/ at the repo root."); + println!("cargo:warning=Build the SvelteKit SPA first: (cd frontend && npm run build)"); + println!("cargo:warning=Or via the workspace shortcut: just fe-build"); + panic!( + "build.rs: missing {}/index.html — see the cargo:warning lines above", + dist.display() + ); + } } // ═══════════════════════════════════════════════════════════════════════════════ diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 00000000..a59d2fd9 --- /dev/null +++ b/clippy.toml @@ -0,0 +1,28 @@ +# Clippy configuration overrides. Kept minimal — each entry documents +# what it's for and when it should be revisited. + +# `clippy::result_large_err` — raise the "big Err variant" ceiling to +# 512 bytes. +# +# Rationale: axum handler signatures shaped as +# `Result` (or `AppError` variants +# that wrap `axum::response::Response`) naturally exceed the default +# 128-byte threshold. `Response` carries a `HeaderMap` (~256 B inline) +# + status + body + extensions; a handful of handlers land in that +# range without doing anything wrong. Fighting the lint per-handler +# with `#[allow]` on every one is churn for zero runtime benefit — +# these Results are constructed on the stack once per request and +# never nested in a hot inner loop. +# +# 512 B keeps the lint's protective value: it still fires on genuinely +# oversized Err variants (embedded `Vec` blobs, avatar payloads, +# large enum aggregates) that WOULD be worth boxing. +# +# Revisit if: +# * A future refactor slims axum Response OR extracts a small error +# enum with an IntoResponse impl across the handler layer — then +# drop this override back to the default 128. +# * A specific handler exceeds 512 B and clippy re-fires — deal with +# that handler individually (boxed error / small enum) rather than +# raising the ceiling further. +large-error-threshold = 512 diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index fbd9da52..d0daecd3 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -170,6 +170,7 @@ export default defineConfig({ { text: "Storage Quotas", link: "/architecture/storage-quotas" }, { text: "Backend Storage", link: "/architecture/backend-storage" }, { text: "File and Blob lifecycle", link: "/architecture/file-and-blob-lifecycle" }, + { text: "Derived and attached blobs", link: "/architecture/derived-and-attached-blobs" }, { text: "ReBAC & Authorization", link: "/architecture/rebac-authorization" }, { text: "User lifecycle", link: "/architecture/user-lifecycle" }, { text: "Authentication model", link: "/architecture/auth-model" }, diff --git a/docs/architecture/backend-storage.md b/docs/architecture/backend-storage.md index cf22ff17..544ac22f 100644 --- a/docs/architecture/backend-storage.md +++ b/docs/architecture/backend-storage.md @@ -26,7 +26,7 @@ probe, and lifecycle behaviour are uniform. Every entry is declared in `OXICLOUD_STORAGE_ENTRIES` (comma-separated list of names). The active entry is stored in `admin_settings` and -switched via `oxicloud --select-storage ` on the command line +switched via `oxicloud storage select ` on the command line or automatically at the end of a successful `backend_migration`. Non-active entries stay reachable through the multi-entry API (test, audit, migrate-into). @@ -119,7 +119,7 @@ Rendered visually via `xxd -l 15 `: Fingerprints are rendered the same colon-hex form (`15:f3:…:50`) everywhere they appear: boot log, admin panel pair chain, `xxd` -inspection, `oxicloud --fingerprint ` CLI, and the rotate / +inspection, `oxicloud storage fingerprint ` CLI, and the rotate / migration audit lines. That means an admin can cross-reference by eye — same string means same key. @@ -234,27 +234,38 @@ can be safely dropped. --- -## 4. Blob consistency (`blobs_consistency`) +## 4. Blob consistency — two jobs, split by what they read -Read-only recoverable job that walks `storage.blobs` and reports -divergence between the DB registry and the physical backend. +The registry side and the physical side are separate tenants. They +used to be one, with `blobs_consistency` probing the backend once per +row; that probe found strictly less than the merge-join below, at N +round-trips instead of one enumeration, so it was removed. -### Shallow mode (default) +### `blobs_consistency` — database only -Per row: +Walks `storage.blobs` and compares `ref_count` against the reference +count computed from `storage.files.blob_hash` + +`chunk_manifests.chunk_hashes[]`. On mismatch: `refcount_mismatch` +(severity `inconsistent`), repairable under `?repair=true`. -- `blob_exists(hash)` on the active backend → if false, record - `blob_missing_from_backend` (severity `data_loss`) -- Compare `ref_count` against the actual reference count computed - from `SUM` over `storage.files.blob_hash` + `chunk_manifests.chunk_hashes[]` - → if mismatch, record `refcount_mismatch` (severity `inconsistent`) +It opens no backend and makes no network call. `?storage=` and +`?deep=true` are inert. Cost is one aggregate SQL per row. -Cost: one existence probe + one aggregate SQL per row. Fast on -S3/Azure (single HEAD). +### `backend_consistency` — everything physical -### Deep mode (`?deep=true`) +Merge-joins the backend's enumeration against `storage.blobs`, both +ordered by hash, yielding both deltas in one pass: -Adds a full read of every blob: +- bytes with no registry row → `orphan_blob` (severity `inconsistent`) +- a registry row with no bytes → `blob_missing_from_backend` + (severity `data_loss`) + +`?storage=` scopes it to any declared entry rather than the live +backend. + +### Deep mode (`?deep=true`, on `backend_consistency`) + +For every hash present on both sides, adds a full read: - Stream the blob through `EncryptedBlobBackend::get_blob_stream` (strips header, decrypts if needed, applies BLAKE3 rescue for @@ -336,7 +347,7 @@ readonly (source stays active — writes safe there), and returns `RunOutcome::Failed`. Operator inspects findings, then either retries (walk short-circuits on head-format matches → cheap re-attempt), fixes the source, or explicitly accepts the partial -via `oxicloud --select-storage `. +via `oxicloud storage select `. --- diff --git a/docs/architecture/caching.md b/docs/architecture/caching.md index ed93aae6..7e2ba448 100644 --- a/docs/architecture/caching.md +++ b/docs/architecture/caching.md @@ -1,8 +1,13 @@ # Caching Architecture -OxiCloud uses **moka** (a lock-free, concurrent cache) for write-behind caching that delivers sub-millisecond hot reads. +OxiCloud runs two independent cache layers with different jobs: -## Cache Layers +1. **In-memory metadata caches** — [moka](https://docs.rs/moka) instances that sit in front of PostgreSQL and other hot-path lookups. Sub-millisecond hits, bounded by entry count, TTL-evicted. Cover file metadata, directory listings, blob hashes, audio metadata, small thumbnails, on-the-fly image transcodes. +2. **On-disk blob cache** — an LRU cache of blob **bytes** on local SSD, only meaningful when the storage backend is remote (S3, Azure). Turns remote fetches into local reads for hot content; bounded by a disk-budget in bytes, LRU-evicted. Off by default. + +The two layers are orthogonal — the moka caches shave query round-trips regardless of backend; the disk-blob cache shaves network round-trips when the backend is elsewhere. + +## Layer 1 — In-memory metadata caches (moka) | Cache | TTL | Max Entries | Purpose | |---|---|---|---| @@ -13,21 +18,63 @@ OxiCloud uses **moka** (a lock-free, concurrent cache) for write-behind caching | Blob hash | 30 s TTI | 5 000 | BLAKE3 hashes for dedup lookups | | Audio metadata | — | 2 000 | ID3 tags and duration | -## How It Works +### How it works 1. **Read path:** check cache → if hit, return immediately (sub-ms); if miss, query PostgreSQL, populate cache, return 2. **Write path:** update PostgreSQL → invalidate relevant cache entries 3. **TTL expiry:** entries are evicted after their time-to-live, ensuring eventual consistency -## Why moka? +### Why moka? - **Lock-free** — no mutex contention under concurrent access - **Bounded memory** — max entries prevent unbounded growth - **TTL + TTI** — supports both time-to-live and time-to-idle eviction - **Async-ready** — works natively with Tokio +## Layer 2 — On-disk blob cache + +A local-SSD LRU cache of blob bytes, sitting between OxiCloud and remote storage backends (S3, Azure, or any other `BlobStorageBackend`). Every blob read probes the local cache first; misses fetch from the remote backend and populate the cache. Writes go to the remote backend AND the local cache simultaneously, so a just-uploaded blob is immediately hot for its own re-reads. + +Structurally: the bytes live on disk, one `.blob` file per hash, sharded by hash prefix under a configurable directory (default `{root}/.blob-cache//.blob`). The in-process index is a `moka::sync::Cache` with a byte-weigher — same crate as Layer 1, but weighing by content size not entry count, and only tracking file existence, not payload. + +### When it earns its keep + +Turn on for any deployment where the backend is not on the same box: + +- S3 (AWS, DigitalOcean Spaces, Cloudflare R2, MinIO on another host, …) +- Azure Blob Storage +- Any future network-attached backend + +Local backends (`LocalFilesystem`) don't need it — they're already on the same box. Enabling it there just doubles disk usage for zero latency win. + +**Thumbnails are the strongest reason to turn this on.** OxiCloud stores thumbnails as blobs alongside primary content (via `content_derived_blobs`, tracked in `storage.blobs` like any other blob) — the sidecar-on-disk layout is gone. On a remote backend this means every thumbnail render is a network fetch: a photos grid with 100 thumbnails is 100 S3 requests, per user, per visit. With Layer 2 on, that cost is paid once per thumbnail hash; every subsequent grid render is local-disk reads. + +Concrete impact for the photos / file-listing hot paths: + +- **Cold render** (all thumbnails uncached): one remote fetch per thumbnail, latency dominated by the backend's per-request round-trip (S3 typically 30-80 ms per object, more at distance). +- **Warm render** (thumbnails cached): local `open()` + read, sub-millisecond per file. +- **Hit rate in practice**: high — thumbnails are small (typically 5-30 KB per size variant), users re-visit the same folders repeatedly, and the LRU pattern strongly favours recency. + +Rule of thumb: if your backend is remote AND you have any user-facing photo grid or file browser, Layer 2 is worth the disk budget. On S3 backends it's the difference between a snappy gallery and a spinner-per-tile browsing experience. + +### Sizing guidance + +The cache is LRU on a disk-budget basis. A working set larger than the cache size will still work but re-fetch cold blobs from the remote — no correctness cost, just latency. Rough sizing: + +- **Home / personal cloud** — 5-10 GB is plenty; the working set for a household of active users is small. +- **Small team / SMB** — 50-100 GB for a hot photo library or shared document store. +- **Large deployment** — size against your top-decile access pattern; the cache doesn't need to cover the whole store. + +The default budget is 50 GB (only applied if the cache is enabled). Adjust to what your local SSD can spare. + +### Interaction with the moka layer + +Independent. A file-metadata hit in Layer 1 tells you the row exists and has a `blob_hash` — but reading the actual bytes still goes through Layer 2 (or straight to the remote backend if disabled). A hit in Layer 2 short-circuits the network fetch; a miss populates it for the next read. + ## Configuration +### In-memory metadata caches (Layer 1) + Cache parameters are currently hardcoded in `src/common/config.rs`. Key defaults: ```rust @@ -35,3 +82,15 @@ file_cache_ttl_ms: 60_000, // 1 minute directory_cache_ttl_ms: 120_000, // 2 minutes max_cache_entries: 10_000, ``` + +### On-disk blob cache (Layer 2) + +Environment-tunable — off by default; enable per deployment when the backend is remote: + +| Env var | Default | Purpose | +|---|---|---| +| `OXICLOUD_STORAGE_CACHE_ENABLED` | `false` | Master switch. Set `true` to wrap the blob backend with the cache decorator. | +| `OXICLOUD_STORAGE_CACHE_MAX_SIZE` | `53687091200` (50 GB) | Disk-budget in bytes. LRU eviction fires when the cache exceeds this size. | +| `OXICLOUD_STORAGE_CACHE_PATH` | `{root}/.blob-cache` | Where the cache files live. Point at a fast SSD; can be a separate volume from the primary storage root. | + +Restart the server after changing any of these — the cache is instantiated once at boot around the configured blob backend. diff --git a/docs/architecture/derived-and-attached-blobs.md b/docs/architecture/derived-and-attached-blobs.md new file mode 100644 index 00000000..19825431 --- /dev/null +++ b/docs/architecture/derived-and-attached-blobs.md @@ -0,0 +1,284 @@ +# Derived and attached blobs + +Two tables hang small artifacts off the blob store: thumbnails, +transcodes, uploaded previews. They look almost identical, and the +difference between them is a security boundary rather than a style +choice. + +("Satellite tables" is the shorthand used in the code and in +`satellites_consistency`, the job that walks both. It is a useful +collective noun once you know what it covers; this page is what it +covers.) + +- **`storage.content_derived_blobs`** — things the *server derived from + file content*. Keyed by the BLAKE3 of the source. +- **`storage.file_attached_blobs`** — things a *user attached to one + specific file*. Keyed by `file_id`. + +Both point into the same blob store underneath (see +[Backend Storage](./backend-storage.md)). The keying is what separates +them. + +```mermaid +erDiagram + FILES ||--o{ ATTACHED : "file_id — per FILE" + FILES }o--|| BLOBS : "blob_hash (its content)" + BLOBS ||--o{ DERIVED : "source_hash — per CONTENT" + DERIVED }o--o| ARTIFACT : "blob_hash (NULL = negative)" + ATTACHED }o--|| ARTIFACT : "blob_hash" + + FILES { + uuid id PK + text blob_hash + } + DERIVED { + text source_hash PK + text kind PK + text variant PK + text blob_hash "NULL = not worth deriving" + text content_type "NULL iff blob_hash NULL" + } + ATTACHED { + uuid file_id PK + text kind PK + text variant PK + text blob_hash + uuid uploaded_by "no FK; nil = imported" + } + ARTIFACT { + text hash PK + } +``` + +Read the two arrows into `ARTIFACT`: `DERIVED` reaches it from +**content**, `ATTACHED` from a **file**. Everything below follows from +that. + +## Why two tables and not one with a `kind` column + +Content keying means identical bytes share one derivation. Upload the +same photo twice and the server renders its thumbnail **once** — both +files resolve to the same `source_hash`, find the same row, and serve +the same blob. That is the entire point for server-derived artifacts: +the derivation is a pure function of the content, so sharing it is +free and correct. + +Apply the same keying to *user-supplied* bytes and it becomes an +attack. If uploaded previews were content-keyed, uploading a file whose +content matches someone else's would let you replace the preview they +see — or read yours in place of theirs. The preview is not derived from +the content; it is an assertion *about* a file, made by whoever uploaded +it, and two people can hold different assertions about identical bytes. + +A single table with a `kind` discriminator could not express this. The +key would have to be one thing or the other, and either choice is wrong +for half the rows. The split is the enforcement mechanism, not a +filing convenience — which is why `thumb_derived_import` explicitly +refuses `ext-` filenames and `thumb_attached_import` explicitly refuses +hash-named ones, rather than one job handling both trees. + +## `storage.content_derived_blobs` + +| column | type | notes | +|---|---|---| +| `source_hash` | `VARCHAR(64)` | PK. BLAKE3 of the **source** content. Dependent reference — holds no refcount; the row is reaped with its source. | +| `kind` | `TEXT` | PK. `thumbnail` \| `transcode` (CHECK-constrained). | +| `variant` | `TEXT` | PK. Opaque rendering discriminator — see [Variant](#variant-holds-every-axis-that-can-vary). | +| `blob_hash` | `VARCHAR(64)` | The derived artifact, **or NULL** — see [Negative rows](#negative-rows). Reference **holder** when present. | +| `content_type` | `TEXT` | MIME of the artifact. NULL exactly when `blob_hash` is NULL. | +| `created_at` | `TIMESTAMPTZ` | | + +A CHECK keeps `blob_hash` and `content_type` NULL together: a type +without bytes describes nothing, and bytes without a type cannot be +served. + +## `storage.file_attached_blobs` + +| column | type | notes | +|---|---|---| +| `file_id` | `UUID` | PK. FK to `storage.files` **ON DELETE CASCADE**. | +| `kind` | `TEXT` | PK. `preview` \| `subtitle` \| `cover_art` (CHECK-constrained). | +| `variant` | `TEXT` | PK. | +| `blob_hash` | `VARCHAR(64)` | `NOT NULL` — there is no negative case here. | +| `content_type` | `TEXT` | `NOT NULL`. | +| `uploaded_by` | `UUID` | `NOT NULL`, and deliberately **no FK**. | +| `created_at` | `TIMESTAMPTZ` | | + +`uploaded_by` follows the provenance convention: an FK with +`ON DELETE SET NULL` would erase the audit trail exactly when it matters +most, and without an `ON DELETE` clause it would block deleting a user +at all. Deleting the uploader must not rewrite history, so the id is +kept even once it no longer resolves. Rows created by the migration +carry the all-zeros sentinel — "imported, uploader unknown" — rather +than a fabricated owner such as the file's `created_by`, which could +later be misread as evidence that someone replaced a preview. + +## Negative rows + +`content_derived_blobs.blob_hash` is nullable, and a NULL row means: +**this derivation was attempted and is known not to be worth storing +for this content.** + +The case that motivated it: `ImageTranscodeService` can only discover +that WebP comes out *larger* than the original by doing the full decode +and re-encode. Without a record, every request repeats that work to +throw the result away. The same applies to a source that cannot be +decoded, or one over the decode ceiling. + +Only failures **deterministic in the content** may be recorded. A +timeout, a closed semaphore, an I/O error reading the source are +properties of the moment, not the bytes; persisting one marks a +perfectly good image as underivable forever, with nothing to retry it. +The asymmetry sets the default — a wrongly-cached transient is silent +and permanent, a missing negative merely costs repeated work — so **when +in doubt, do not write the row.** + +A sentinel hash was considered and rejected: it would stop `blob_hash` +naming a real blob, and every consumer joining on it would need to learn +the exception or silently mishandle it. NULL is already SQL's way of +saying "no blob", and joins drop it naturally. + +`file_attached_blobs` has no negative case. There is nothing to attempt +— the bytes either arrived from a client or they did not. + +### The NULL trap + +This has caused two bugs, both found before shipping, and it will cause +more. SQL comparison against NULL yields NULL, so: + +```sql +EXISTS (SELECT 1 FROM storage.blobs b WHERE b.hash = d.blob_hash) +``` + +is **false** for every negative row. Whether that is right depends +entirely on what you are asking: + +- **Refcounts — correct.** A negative row holds no reference, so it must + not contribute. `content_derived_ref_sql` relies on exactly this. +- **Dangling checks — wrong.** `satellites_consistency` reported every + negative row as `derived_dangling_blob` at `data_loss` severity: a row + correctly pointing at nothing, reported as an artifact that had gone + missing. It needs `d.blob_hash IS NULL OR `. +- **Enumeration — wrong, and it fails loudly.** + `list_referenced_blobs` decodes `blob_hash` into `String`; the first + NULL takes the whole sweep down. It needs `WHERE blob_hash IS NOT NULL`. + +Anything joining on `blob_hash` has to decide which of these it is. + +## `variant` holds every axis that can vary + +`variant` is an opaque discriminator, and the schema states the rule: +new axes go **inside this string, never into new columns**. + +The two tables therefore look asymmetric, and correctly so: + +| table | variant | why | +|---|---|---| +| `content_derived_blobs` | `icon.webp`, `preview.jpg` | size **and** format | +| `file_attached_blobs` | `icon` | size only | + +A single source legitimately has two thumbnails at one size — WebP for +capable clients, JPEG for the rest — and since the PK is +`(source_hash, kind, variant)`, the format must be inside `variant` or +those rows collide and only one can exist. Uploaded previews have no +format axis: `store_external_thumbnail` re-encodes to JPEG on write, so +`image/jpeg` is a constant and `.jpg` in every variant would carry no +information. + +**Rejected alternative: key on `content_type` instead.** Making +`content_type` part of the key would express the same thing, and it was +reasonable until negative rows landed. It is now foreclosed — +`content_type` must be nullable for negative rows, and PostgreSQL does +not allow a nullable column in a primary key. A UNIQUE constraint would +not rescue it either: NULLs compare as *distinct* in a unique index, so +duplicate negative rows for one `(source_hash, kind, variant)` would +become possible, and that row's singularity is what the mechanism +depends on. Two softer objections stand regardless — `content_type` is a +presentation value, and MIME strings are not canonical (`image/jpg` and +`image/jpeg` name one format). + +**If the assumption changes**, the migration has a known shape. +`20261022000000_derived_variant_encodes_format.sql` did it once for the +derived table: append the format to existing variants, update the +callsites that build the string. Nothing is lost in the meantime — +`content_type` records the real format — it simply is not part of the +key, so two formats cannot coexist until it is. + +## Worked examples + +**The same image uploaded twice.** Two `storage.files` rows, one +`blob_hash` between them, **one** `content_derived_blobs` row per +`(kind, variant)`, one thumbnail blob. The second upload renders +nothing; it finds the existing row. Copying either file adds no row at +all — the copy shares the source hash, so it resolves to the same +derivation. + +**A PDF with a client-uploaded preview.** One `file_attached_blobs` row +keyed by that `file_id`. Copy the file and the row is **duplicated** for +the new id (`storage.copy_file_satellites`), because the preview belongs +to the file, not to the content. Without that duplication the copy +silently loses its preview — and a PDF has no server-side render path, +so nothing regenerates it. + +**A screenshot WebP cannot shrink.** One row, `blob_hash` and +`content_type` both NULL. A reader concludes: the transcode was +attempted, it is known not to help *for this content*, serve the +original and do not retry. Every file sharing those bytes inherits the +verdict. + +**One source, thumbnailed and transcoded.** Two rows, same +`source_hash`, `kind` of `thumbnail` and `transcode`. Add a JPEG +fallback thumbnail and it is a third row, differing only in `variant` +(`icon.webp` vs `icon.jpg`). + +## Lifecycle + +**References.** A positive `blob_hash` is a reference *holder* — it +bumps `chunk_manifests.ref_count` through `DedupService::add_reference`, +so `dedup_gc` cannot reap an artifact a satellite still points at. A +negative row holds none. `source_hash` is a *dependent* reference and +holds nothing. + +**Reaping.** Derived rows are removed by `purge_derived_blobs` when +their source blob is reaped. Attached rows vanish by +`ON DELETE CASCADE` when their file is deleted — which happens **inside +the database**, where the Rust lifecycle hooks cannot observe it, so +`trg_file_attached_blobs_decrement_blob_ref` releases the blob reference +on `DELETE`. The trigger fires on DELETE only; replacing a preview +updates `blob_hash` in place and the Rust path handles that reference +swap. + +**Writing a derived row requires its source to exist.** +`store_derived_blob` guards the insert with an `EXISTS` on +`chunk_manifests`/`blobs`. Without it, a row written just after its +source was reaped would pin its artifact forever: nothing would ever +reap that `source_hash` again, so `purge_derived_blobs` could never +fire. That is not hypothetical — it shipped once, as a permanent blob +leak. + +**Consistency coverage.** `satellites_consistency` is the only job that +walks these tables, and it exists because of a gap the others cannot +close: a satellite row whose *source* is gone breaks no invariant any +other check looks at. The reference is valid, the refcount is correct, +the bytes are present — every Blob-centric job agrees the system is +healthy while the artifact is pinned forever. Blob-side integrity +(missing bytes, orphans, bit-rot) belongs to `backend_consistency`; +refcount arithmetic to `blobs_consistency` and +`manifests_consistency`. + +## Adding a third artifact type + +Ask one question first: **is it derived from the content, or asserted +about a file?** + +Derived from content — a waveform, an extracted page count, an OCR +layer — goes in `content_derived_blobs` under a new `kind`, and shares +across identical content for free. + +Supplied by a user — a custom cover image, a hand-authored subtitle +track — goes in `file_attached_blobs`, and must be duplicated on copy +rather than shared. + +Getting that backwards is not a performance mistake. Putting +user-supplied bytes in the content-keyed table means one user's upload +is served to everyone whose file happens to match. diff --git a/docs/architecture/index.md b/docs/architecture/index.md index 62f12d10..2ed52f5d 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -75,4 +75,5 @@ src/ - [Resource Listing API →](/architecture/resource-listing) - [Storage Quotas →](/architecture/storage-quotas) - [Backend Storage →](/architecture/backend-storage) +- [Derived and Attached Blobs →](/architecture/derived-and-attached-blobs) — thumbnails, transcodes and uploaded previews: why content-keyed and file-keyed artifacts need separate tables - [Background Jobs →](/architecture/jobs) diff --git a/docs/config/admin-settings.md b/docs/config/admin-settings.md index 0dc1f082..e80b6820 100644 --- a/docs/config/admin-settings.md +++ b/docs/config/admin-settings.md @@ -93,19 +93,23 @@ Runs are recoverable — status, cursor, and per-blob failure findings all live If an entry is renamed or removed from `.env` while the DB pointer still names the old one, boot aborts with a clear error pointing at: ``` -oxicloud --select-storage +oxicloud storage select ``` This one-shot repair command re-runs the same env-parse the server does at boot, verifies `` is declared in `OXICLOUD_STORAGE_ENTRIES`, updates `admin_settings.storage.active_backend_name` in the DB, and exits. Operator then restarts normally. See [Environment Variables — Storage Entries](/config/env#storage-entries-multi-entry-recommended) for the model, and [`oxicloud --help`](https://github.com/oxicloud/oxicloud/blob/main/src/main.rs) for the full flag list. ### Auditing entries other than the active one -`blobs_consistency` and `backend_consistency` (recoverable jobs on the Jobs tab) accept `?storage=` to probe any declared entry — not just the live one. Use this to verify a migration target before cutover, or to audit an old backend after cutover but before decommissioning: +`backend_consistency` (a recoverable job on the Jobs tab) accepts `?storage=` to audit any declared entry — not just the live one. Use this to verify a migration target before cutover, or to audit an old backend after cutover but before decommissioning: ``` -POST /api/admin/jobs/blobs_consistency/trigger?storage= +POST /api/admin/jobs/backend_consistency/trigger?storage= ``` +Add `?deep=true` to also read every blob back and re-hash it, which catches silent bit-rot. That is a full read of the entry and can take hours. + +`blobs_consistency` does *not* accept `?storage=`: it only reads the database, so there is no entry for it to scope. + Unknown names 400 at the HTTP layer. ## Data Storage diff --git a/docs/config/authentication.md b/docs/config/authentication.md index 967bc0b7..1b8b8e97 100644 --- a/docs/config/authentication.md +++ b/docs/config/authentication.md @@ -135,12 +135,12 @@ Password-using deployments will opt in via three env vars: 2. **`OXICLOUD_AUTH_OPAQUE_SERVER_SETUP`** — generated once and persisted like your JWT secret. Rotating this invalidates every user's registration; treat it as one of the crown jewels. Two ways to generate: ```bash # Docker (recommended in production — no toolchain needed): - docker run --rm ghcr.io/atalayalabs/oxicloud:latest oxicloud-cli opaque setup + docker run --rm ghcr.io/atalayalabs/oxicloud:latest oxicloud opaque setup # From a source checkout: - cargo run --bin oxicloud-cli -- opaque setup + cargo run --bin oxicloud -- opaque setup ``` - Both print the base64 value on stdout (with guidance on stderr, so shell pipelines like `$(docker run ... oxicloud-cli opaque setup)` capture cleanly). + Both print the base64 value on stdout (with guidance on stderr, so shell pipelines like `$(docker run ... oxicloud opaque setup)` capture cleanly). 3. **`OXICLOUD_AUTH_OPAQUE_KSF_*`** — client-side Argon2id key-stretching cost. Defaults (46 MiB / 1 iter / 1 lane) match OWASP's interactive-auth recommendation. See the next section for the rationale + when to bump. The `OXICLOUD_HASH_*` variables (server-side legacy Argon2) and `OXICLOUD_AUTH_OPAQUE_KSF_*` (client-side OPAQUE Argon2) are intentionally separate: the server-side path is RAM-bounded by concurrent-login traffic and needs to stay modest; the client-side path is single-user per attempt and can be tuned independently. Tuning them together would force a bad compromise in one direction or the other. diff --git a/docs/config/env.md b/docs/config/env.md index ffc09e5f..0d1e02a8 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -18,6 +18,7 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Direct (non-chunked) uploads stream straight into the blob store and need no spool directory. Placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_REUSE_PORT` | `false` | Enable `SO_REUSEPORT` so multiple processes can share the same port. **Disabled by default** — a second accidental instance will fail with "address already in use". Enable only for deliberate multi-worker setups (process supervisor, rolling restart). Not supported on Windows. | | `OXICLOUD_METRICS_LISTEN` | (unset) | Prometheus `/metrics` listener address (e.g. `127.0.0.1:9090`, IPv6 allowed as `[::1]:9090`). **Unset = disabled**: no `/metrics` endpoint is bound and no metrics recorder is installed (zero runtime cost). When set, a separate HTTP listener on this address serves the text-format scrape. **Deliberately NOT merged into the main API** — no auth, CSRF, or DPoP layer in front. Bind to loopback or a private interface unless you intend to expose metrics publicly. Starter counters: `oxicloud_dpop_verify_failed_total{reason}`, `oxicloud_dpop_proof_missing_total`, `oxicloud_dpop_header_missing_on_bound_session_total`, `oxicloud_dpop_replay_detected_total`, `oxicloud_dpop_nonce_challenges_issued_total`. | +| `OXICLOUD_STARTUP_JOBS` | `thumb_derived_import?repair=true,thumb_attached_import?repair=true,transcode_import?repair=true` | Background jobs dispatched once at boot, comma-separated, each `name` or `name?flag=true` using the same syntax as `POST /api/admin/jobs/{name}/trigger`. Flags: `force`, `deep`, `repair`, `storage`. **The default migrates thumbnails out of the legacy `.thumbnails/` directory and deletes the originals**, so the migration completes without anyone triggering it from the admin panel; each sidecar is read back through the normal stack before it is unlinked, and every deletion is audited. An explicit value **replaces** the default; set it empty (`OXICLOUD_STARTUP_JOBS=`) to disable startup jobs, or to `thumb_derived_import,thumb_attached_import` to import without deleting. **Non-blocking** — readiness never waits on a job; entries run sequentially in the background. **Fail-fast** — an unknown job name or flag panics at boot, because a silently-dropped entry means a migration that never runs. A run interrupted by a restart resumes from its cursor on the next boot, so a long migration finishes across restarts. Safe to leave at the default: the jobs are idempotent, and once drained a run does nothing. See [Thumbnail Migration](./thumbnail-migration.md) for the upgrade runbook. | ## Database @@ -58,7 +59,7 @@ OPAQUE (RFC 9807) is a zero-knowledge password-authenticated key exchange: the p | Variable | Default | Description | |---|---|---| | `OXICLOUD_AUTH_OPAQUE_MODE` | `off` | Runtime mode. `off` = endpoints 404 (default). `migrate` = endpoints live, legacy `POST /api/auth/login` still accepted. `opaque_only` = endpoints live, legacy refused for users with an envelope. **Effective-mode cross-check**: when `password` is not in `OXICLOUD_AUTH_METHODS`, the mode is auto-downgraded to `off` with an audit-channel INFO line (OPAQUE only replaces the password path — nothing to shadow in an OIDC-only or magic-link-only deployment). So OIDC / magic-link-only operators can safely ignore every `OXICLOUD_AUTH_OPAQUE_*` variable. | -| `OXICLOUD_AUTH_OPAQUE_SERVER_SETUP` | — | Base64-encoded `ServerSetup` blob. **Required** when `OXICLOUD_AUTH_OPAQUE_MODE != off` AND password is enabled — the server refuses to start with a helpful error otherwise. Generate once with the `oxicloud-cli opaque setup` subcommand and persist the value like your JWT secret. **Never rotate** — rotating invalidates every user's envelope (they'd all need to reset their passphrase). | +| `OXICLOUD_AUTH_OPAQUE_SERVER_SETUP` | — | Base64-encoded `ServerSetup` blob. **Required** when `OXICLOUD_AUTH_OPAQUE_MODE != off` AND password is enabled — the server refuses to start with a helpful error otherwise. Generate once with the `oxicloud opaque setup` subcommand and persist the value like your JWT secret. **Never rotate** — rotating invalidates every user's envelope (they'd all need to reset their passphrase). | | `OXICLOUD_AUTH_OPAQUE_KSF_MEMORY_KIB` | `47104` | Client-side Argon2id memory cost in KiB (46 MiB — matches OWASP interactive-auth recommendation). Runs on the user's device during OPAQUE login/registration, TWICE per login. Distinct from `OXICLOUD_HASH_MEMORY_COST` (server-side legacy path). Bumping raises brute-force cost after a hypothetical envelope leak but also raises login latency and risks WASM heap OOM on low-memory devices — see `authentication.md § OPAQUE — KSF parameters` for the full rationale + per-device latency table. | | `OXICLOUD_AUTH_OPAQUE_KSF_ITERATIONS` | `1` | Client-side Argon2id iteration count (OWASP interactive-auth recommendation). | | `OXICLOUD_AUTH_OPAQUE_KSF_PARALLELISM` | `1` | Client-side Argon2id parallelism lanes (OWASP recommendation). Higher only helps on multi-core hardware and can hurt single-core / older mobile devices. | @@ -94,6 +95,8 @@ DPoP cryptographically binds a session cookie to a browser-held ECDSA keypair (P | `OXICLOUD_ENABLE_TRASH` | `true` | Trash / recycle bin | | `OXICLOUD_ENABLE_SEARCH` | `true` | Full-text and metadata search | | `OXICLOUD_ENABLE_MUSIC` | `true` | Music playlists and audio metadata | +| `OXICLOUD_ENABLE_VIDEO_THUMBNAILS` | `true` | Server-side single-frame thumbnail extraction from uploaded videos (one frame → WebP). Requires `ffmpeg` on `PATH` (override with `OXICLOUD_FFMPEG_PATH`). When true and ffmpeg is missing at boot, a WARN log is emitted and videos fall back to a placeholder icon. Set to `false` to skip the ffmpeg lookup entirely — useful on hosts where ffmpeg can't be installed, or when the client uploads video previews itself (some desktop/mobile clients generate thumbnails locally and POST them alongside the video). | +| `OXICLOUD_FFMPEG_PATH` | `ffmpeg` (on PATH) | Absolute path to the ffmpeg binary. Ignored when `OXICLOUD_ENABLE_VIDEO_THUMBNAILS=false`. Useful for pinning a specific static build or when ffmpeg lives outside the default PATH. | | `OXICLOUD_EXPOSE_SYSTEM_USERS` | `true` | Expose other OxiCloud users as a read-only address book at `GET /api/address-books` | | `OXICLOUD_GRANT_CLEANUP_ENABLED` | `true` | Background daemon that deletes expired rows from `storage.role_grants`. The authorization engine already filters expired grants out of every permission check at read time (`expires_at IS NULL OR expires_at > NOW()`), so leaving expired rows in place is a hygiene issue — not a security one. This daemon garbage-collects them daily. Set to `false` to keep every expired grant row forever (uncommon; a fresh install rarely wants this). | | `OXICLOUD_GRANT_CLEANUP_GRACE_DAYS` | `15` | Days past a grant's `expires_at` before the row is eligible for deletion. The grace window preserves the audit / support answer to "what happened to my access?" for a couple of weeks past expiration. Values below 1 are legal but discouraged — the recommendation is **≥ 15 days**. Values above the actual grant TTL used by clients waste index space; a few weeks is the sweet spot. | @@ -131,7 +134,7 @@ Each declared name `` then reads its own set of per-entry variables: - A declared name whose required per-entry fields are missing (`_BACKEND` never set, S3 with no `_S3_BUCKET`, Azure with no `_AZURE_CONTAINER`). - Setting `OXICLOUD_STORAGE_ENTRIES` alongside any of the legacy flat vars below (`OXICLOUD_STORAGE_BACKEND`, `OXICLOUD_S3_*`, `OXICLOUD_AZURE_*`, `OXICLOUD_STORAGE_ENCRYPTION_*`). Pick one mode; the error lists every conflicting var to remove. -- A DB pointer (`admin_settings.storage.active_backend_name`) that names an entry not in the current `_ENTRIES`. The error points at the repair flag `oxicloud --select-storage ` — verify + UPDATE DB + exit. +- A DB pointer (`admin_settings.storage.active_backend_name`) that names an entry not in the current `_ENTRIES`. The error points at the repair flag `oxicloud storage select ` — verify + UPDATE DB + exit. **Example** — two entries, local disk plus an S3 target for planned migration: diff --git a/docs/config/index.md b/docs/config/index.md index 35d424f2..c47a6cf1 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -7,6 +7,7 @@ OxiCloud is configured entirely via **environment variables** (no config files n - [Deployment & Docker](/config/deployment) — Docker Compose, Kubernetes Helm chart, image details - [Environment Variables](/config/env) — complete reference of all `OXICLOUD_*` variables - [Storage Fine Tuning](/config/storage-fine-tuning) — sizing the upload caps + spool directories; tmpfs vs real disk; NVMe split layouts +- [Thumbnail Migration](/config/thumbnail-migration) — upgrading past `.thumbnails/`: what runs on first boot, taking a snapshot first, verifying afterwards - [Authentication](/config/authentication) — JWT auth, login, refresh, password changes, and auth status - [OIDC / SSO](/config/oidc) — single sign-on with Keycloak, Authentik, Authelia, Google, Azure AD - [WOPI (Office Editing)](/config/wopi) — Collabora Online / OnlyOffice integration diff --git a/docs/config/thumbnail-migration.md b/docs/config/thumbnail-migration.md new file mode 100644 index 00000000..c9c762a7 --- /dev/null +++ b/docs/config/thumbnail-migration.md @@ -0,0 +1,153 @@ +# Thumbnail migration runbook + +Thumbnails used to live as files under `{STORAGE_PATH}/.thumbnails/`. +They now live in the content-addressed blob store, alongside file +content. This page is for operators upgrading across that change. + +**You do not have to do anything.** The migration runs itself, in the +background, on the first boot after the upgrade. The rest of this page +is for operators who want to verify it, take a safety net first, or +understand what it did. + +## What runs, and when + +Two background jobs, dispatched once at startup and daily thereafter: + +| Job | Migrates | Regenerable if lost? | +|---|---|---| +| `thumb_derived_import` | Thumbnails the server rendered from file content | Yes — the next request re-renders | +| `thumb_attached_import` | Previews a client uploaded (`ext-{file_id}.jpg`) | **No** — there is no render path for these | + +Both import each sidecar into blob storage, read it back to confirm the +copy is byte-identical, and only then delete the original. When the +directory is empty it is removed, and `.thumbnails/` stops existing. + +Startup dispatch is non-blocking — the server is ready immediately and +the migration proceeds behind it. A run interrupted by a restart resumes +from where it stopped, so a large installation finishes over several +restarts rather than starting again each time. + +This is controlled by `OXICLOUD_STARTUP_JOBS`, which defaults to: + +``` +OXICLOUD_STARTUP_JOBS=thumb_derived_import?repair=true,thumb_attached_import?repair=true +``` + +To **import without deleting** — migrate now, inspect, delete later: + +``` +OXICLOUD_STARTUP_JOBS=thumb_derived_import,thumb_attached_import +``` + +The sidecars then stay on disk. Trigger the deletion when you are ready +from **Admin → Jobs**, using each job's Repair action. + +To disable startup jobs entirely, set the variable to an empty value. + +## Taking a safety net first + +Recommended for any installation where the uploaded previews matter, and +cheap enough to be worth it regardless. Both parts must be captured +together — a database that references blobs a storage snapshot predates +is worse than neither. + +**1. Stop the server.** A snapshot taken while writes are in flight can +catch a blob that exists on disk without its database row, or the +reverse. + +```bash +systemctl stop oxicloud # or: docker compose stop oxicloud +``` + +**2. Snapshot the database.** + +```bash +pg_dump --format=custom --file=oxicloud-preflight.dump "$DATABASE_URL" +``` + +Use `--format=custom`; restoring it needs `pg_restore --disable-triggers`, +because the folder table carries a self-referencing foreign key that a +plain SQL restore cannot order correctly. + +**3. Snapshot the storage directory.** At minimum `.thumbnails/`, which +is what the migration touches: + +```bash +tar -czf oxicloud-thumbnails-preflight.tar.gz -C "$STORAGE_PATH" .thumbnails +``` + +A whole-directory snapshot is better if you have the space — filesystem +or volume snapshots (ZFS, LVM, EBS) are ideal, since they are atomic and +near-instant: + +```bash +zfs snapshot tank/oxicloud@preflight +``` + +**4. Start the server.** The migration begins in the background. + +Keep both snapshots until you have run the verification below and are +satisfied. + +## Verifying the migration + +Two checks, both from **Admin → Jobs** or the API. Run them after the +migration reports no remaining work. + +**1. Every mapping points at a blob that exists.** Run +`satellites_consistency`. It walks both thumbnail tables and reports any +row whose blob or source is gone. A clean run means nothing was lost in +the bookkeeping. + +``` +POST /api/admin/jobs/satellites_consistency/trigger +``` + +**2. Every blob still hashes to what it claims.** Run +`backend_consistency` with `?deep=true`. It reads every blob back from +storage and re-hashes it, which covers the migrated thumbnails along +with everything else. This is a full read of your storage and can take +hours on a large installation — schedule it accordingly. + +``` +POST /api/admin/jobs/backend_consistency/trigger?deep=true +``` + +A clean pass on both means the thumbnails are readable, correctly +referenced, and byte-intact in their new home. At that point the +snapshots can be discarded. + +## Checking it finished + +`.thumbnails/` is gone. That is the whole test: + +```bash +ls -d "$STORAGE_PATH/.thumbnails" # No such file or directory +``` + +If you instead find `.thumbnails.migrated/`, the migration completed but +could not remove the directory, because something that is not a +thumbnail was inside it — a `.DS_Store` from macOS Finder is the usual +culprit. The tree was moved aside instead of deleted. Its contents are +no longer used and it is safe to remove by hand once you have looked at +what is in there. + +While either directory is absent, the server skips the legacy read path +entirely, at no cost. While `.thumbnails/` is present, reads fall back +to it on a miss, which is what makes the migration invisible to users +while it runs. + +## If something looks wrong + +Every deletion is written to the audit log, naming the job, the file +removed and the blob that replaced it. To review what a migration +removed: + +```bash +journalctl -u oxicloud | grep sidecar_deleted +``` + +A sidecar is only ever deleted after its replacement has been read back +and compared byte-for-byte, so a file that failed that check is still on +disk. Those show up as findings on the job's run in **Admin → Jobs**, +with the reason recorded per file. diff --git a/docs/guide/backend-storage.md b/docs/guide/backend-storage.md index 19d04bdd..07bafce2 100644 --- a/docs/guide/backend-storage.md +++ b/docs/guide/backend-storage.md @@ -167,13 +167,13 @@ If you rename or remove a backend from `.env` while it was still the active one, ``` active_backend_name = `s3_prod`, but no entry with that name is declared in OXICLOUD_STORAGE_ENTRIES. Available: [local_main]. […] -oxicloud --select-storage +oxicloud storage select ``` Run the command it suggests to pick a still-declared backend and the server will boot again on the next start: ``` -oxicloud --select-storage local_main +oxicloud storage select local_main ``` This just updates which backend OxiCloud considers active — it doesn't move any data. diff --git a/docs/install/binary.md b/docs/install/binary.md new file mode 100644 index 00000000..9d29b139 --- /dev/null +++ b/docs/install/binary.md @@ -0,0 +1,245 @@ +# Installing OxiCloud from a Binary Release + +OxiCloud ships prebuilt binaries for common Linux and macOS platforms +attached to every tagged release on GitHub. This page covers downloading, +verifying, and running one. + +If you'd rather run OxiCloud as a container, see the Docker image at +`ghcr.io/atalayalabs/oxicloud`. If you're a Rust developer who just +wants the binary without hand-fetching a tarball, `cargo binstall +oxicloud` picks the right archive for your host automatically. + +## Which tarball do I want? + +Every release attaches three tarballs plus a `SHA256SUMS` manifest. +Pick by your host's architecture and OS: + +| Host | Tarball | +|---|---| +| Linux x86-64 (Intel / AMD servers, most VPS, WSL) | `oxicloud--x86_64-unknown-linux-musl.tar.gz` | +| Linux ARM64 (Raspberry Pi 4/5, Ampere, Graviton, ARM servers) | `oxicloud--aarch64-unknown-linux-musl.tar.gz` | +| macOS Apple Silicon (M-series) | `oxicloud--aarch64-apple-darwin.tar.gz` | + +The Linux tarballs link against musl, so they run on ANY glibc version +— Alpine, Debian, Ubuntu, Fedora, Arch, Rocky, and every version in +between. You never need to worry about `GLIBC_x.yy not found`. + +**Intel macOS, Windows, and 32-bit ARM are not currently shipped as +prebuilt tarballs.** Intel Mac users have three fallbacks: + +1. `cargo install oxicloud --locked --features bundled-assets` from + source (needs the Rust toolchain). +2. Docker: `docker pull --platform linux/amd64 ghcr.io/atalayalabs/oxicloud`. +3. Run one of the two Linux musl tarballs inside a Linux VM + (Multipass, Lima, UTM, etc.). + +## Hardware notes + +| Model | Notes | +|---|---| +| Pi 5 (4 GB / 8 GB) | Good experience | +| Pi 4 (4 GB / 8 GB) | Solid | +| Pi 4 (2 GB) | Works with face indexing disabled; expect swap under load | +| Pi 3 (any variant) | Marginal — only for a very light single-user personal cloud | +| Pi 2 / Pi Zero / Pi 1 | Not supported (1 GB RAM is below the practical floor) | +| Any ARM64 server | Good — the aarch64 tarball is what you want | +| Any x86-64 server from 2010 or newer | Good — Nehalem / Bulldozer + newer, per the release CPU baseline | + +## Verifying the download + +Every release ships a `SHA256SUMS` manifest listing every tarball with +its hash. Verify your download before extracting: + +``` +sha256sum -c SHA256SUMS +``` + +Only files present in the current directory are checked, so this +succeeds when just the tarball you downloaded matches its entry. + +## Extracting + +The archive lands as a per-version-per-triple directory next to it: + +``` +tar xzf oxicloud--.tar.gz +cd oxicloud--/ +ls +# oxicloud example.env LICENSE README-install.md +``` + +The four files: + +- `oxicloud` — the single self-contained binary. The server, all + operator subcommands (`oxicloud opaque setup`, `oxicloud migrate + nfc-filenames`, `oxicloud storage select`), and the SvelteKit web + frontend are all baked in. +- `example.env` — every OxiCloud environment variable documented with + defaults. Copy to `.env` and edit as needed. +- `LICENSE` — the project license. +- `README-install.md` — a shorter version of this page for offline + reference. + +## Prerequisites + +Only one moving part is required: a PostgreSQL 13+ instance with the +`pg_trgm` and `ltree` extensions available. Anything else you might +need is either baked into the binary or optional. + +### Required + +- **PostgreSQL 13+** with `pg_trgm` and `ltree` extensions. Any distro + package works (Debian/Ubuntu's `postgresql`, Alpine's `postgresql`, + Homebrew's `postgresql@17`, etc.). Cloud databases like Neon, + Supabase, and RDS also work provided the two extensions are enabled. + +### System libraries (usually pre-installed) + +- **`ca-certificates`** — for outbound HTTPS (OIDC discovery, S3, magic + links). Pre-installed on essentially every distribution. +- **`tzdata`** — timezone database. Pre-installed on nearly every + distribution; alpine minimal images sometimes need it added. + +### Optional + +- **`ffmpeg`** — only needed if you want the server to extract a + thumbnail frame from uploaded videos. When ffmpeg is missing the + server logs a warning at boot and videos get a placeholder icon — + everything else keeps working. If your client uploads video + previews itself (some desktop and mobile clients do), or if you + simply don't want thumbnails, set + `OXICLOUD_ENABLE_VIDEO_THUMBNAILS=false` in your `.env` to silence + the warning. + +Distro install commands for the optional prerequisite: + +| Distro | Command | +|---|---| +| Alpine | `apk add ffmpeg` | +| Debian / Ubuntu | `apt install ffmpeg` | +| Fedora / RHEL | `dnf install ffmpeg` (RPM Fusion for the full codec set) | +| Arch | `pacman -S ffmpeg` | +| macOS | `brew install ffmpeg` | +| Any Linux (portable) | grab a static build from https://github.com/BtbN/FFmpeg-Builds/releases and point `OXICLOUD_FFMPEG_PATH` at it | + +## First run + +The absolute minimum to boot the server is `DATABASE_URL`: + +``` +DATABASE_URL="postgres://oxicloud:secret@localhost:5432/oxicloud" \ + ./oxicloud +``` + +The binary applies its embedded database migrations on startup, then +listens on `127.0.0.1:8086` by default. Open your browser at +`http://localhost:8086/` and follow the setup flow to create the first +admin account. + +For anything more than a smoke test, copy `example.env` to `.env`, +edit it, and run `./oxicloud --config .env` — that pins the config +source and makes stray shell environment variables not silently leak +in. + +## Running as a systemd service (Linux) + +Move the binary to a system location and create a systemd unit. The +example below runs as a dedicated `oxicloud` user, loads config from +`/etc/oxicloud/oxicloud.env`, and stores data under `/var/lib/oxicloud`. + +``` +sudo useradd --system --home /var/lib/oxicloud --create-home --shell /usr/sbin/nologin oxicloud +sudo install -m 0755 oxicloud /usr/local/bin/oxicloud +sudo mkdir -p /etc/oxicloud +sudo cp example.env /etc/oxicloud/oxicloud.env +sudo chown -R oxicloud:oxicloud /etc/oxicloud +sudo chmod 0640 /etc/oxicloud/oxicloud.env +``` + +Create `/etc/systemd/system/oxicloud.service`: + +``` +[Unit] +Description=OxiCloud self-hosted cloud storage +After=network-online.target postgresql.service +Wants=network-online.target + +[Service] +Type=simple +User=oxicloud +Group=oxicloud +WorkingDirectory=/var/lib/oxicloud +ExecStart=/usr/local/bin/oxicloud --config /etc/oxicloud/oxicloud.env +Restart=on-failure +RestartSec=5 + +# Sandbox — plenty of room to tighten further per your policy +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/oxicloud +PrivateTmp=true + +[Install] +WantedBy=multi-user.target +``` + +Enable and start: + +``` +sudo systemctl daemon-reload +sudo systemctl enable --now oxicloud +sudo systemctl status oxicloud +journalctl -u oxicloud -f +``` + +Terminate the reverse-proxy (nginx, Caddy, HAProxy, Traefik) in front +of it for TLS and public exposure — OxiCloud itself binds plaintext +HTTP on `127.0.0.1` by default. + +## Upgrading + +Replace the binary and restart the service: + +``` +# Download and verify the new tarball +sha256sum -c SHA256SUMS +tar xzf oxicloud--.tar.gz +cd oxicloud--/ + +sudo systemctl stop oxicloud +sudo install -m 0755 oxicloud /usr/local/bin/oxicloud +sudo systemctl start oxicloud +``` + +Database migrations apply automatically on startup. Rollbacks are not +supported by sqlx's migration model; if you need to roll back, stop +the server, roll back your Postgres data directory to a snapshot, and +install the previous binary. + +## Installing via `cargo binstall` + +If you already have the Rust toolchain and just want the binary +without hand-picking a tarball: + +``` +cargo binstall oxicloud +``` + +`cargo-binstall` reads the URL template baked into the release +metadata, downloads the tarball for your host triple, verifies its +signature (when present), and installs `oxicloud` into +`~/.cargo/bin`. This resolves to the same tarball you'd download by +hand. + +## Where to go from here + +- Environment reference — see [`docs/config/env.md`](../config/env.md) + for every `OXICLOUD_*` variable and its default. +- Authentication setup (OPAQUE, OIDC, magic links) — see + [`docs/config/authentication.md`](../config/authentication.md). +- Storage backends (local disk, S3, Azure Blob, encryption) — see + [`docs/config/storage.md`](../config/storage.md) if present, or the + entries under `OXICLOUD_STORAGE_*` in the environment reference. +- File a bug or a feature request — GitHub issues at + https://github.com/AtalayaLabs/OxiCloud. diff --git a/docs/plan/benchmake-and-performance-tracking.md b/docs/plan/benchmake-and-performance-tracking.md index 46f40f17..d4bcdc39 100644 --- a/docs/plan/benchmake-and-performance-tracking.md +++ b/docs/plan/benchmake-and-performance-tracking.md @@ -41,7 +41,9 @@ Mirrors the `tests/api/` shell pattern (run.sh, test.env, separate server port). ### 2. Rust bulk seeder — `src/bin/load-seed.rs` -New binary registered in `Cargo.toml` alongside `generate-openapi` and `migrate-nfc-filenames`. +New binary registered in `Cargo.toml` alongside `generate-openapi` +(the historical `migrate-nfc-filenames` bin has since been folded into +`oxicloud migrate nfc-filenames` — see `docs/plan/bundled-binary.md` § 1b). **CLI:** ``` @@ -158,7 +160,7 @@ Matches existing recipe naming (`test-*`, `front-*`, `api-test`). - `.github/workflows/load-nightly.yml`, `load-smoke.yml` **Modify:** -- `Cargo.toml` — add `[[bin]] name = "load-seed" path = "src/bin/load-seed.rs"` after the `migrate-nfc-filenames` entry +- `Cargo.toml` — add `[[bin]] name = "load-seed" path = "src/bin/load-seed.rs"` after the `generate-openapi` entry (the `migrate-nfc-filenames` bin referenced in earlier drafts has been folded into `oxicloud migrate nfc-filenames`) - `justfile` — append four `load*` recipes - `.gitignore` — add `tests/load/results/*.json` and `tests/load/storage/` diff --git a/docs/plan/bundled-binary.md b/docs/plan/bundled-binary.md new file mode 100644 index 00000000..d72f790a --- /dev/null +++ b/docs/plan/bundled-binary.md @@ -0,0 +1,888 @@ +# Bundled Binary Distribution — Multi-Platform Plan + +## Context + +Users have asked for a way to run OxiCloud without Docker — a plain +binary. Today `release.yml` only creates a GitHub Release with notes; +no binary is attached. The Docker workflow (`docker-publish.yml`) ships +multi-arch images, but that's a separate audience. + +The blocker for a "just download and run" experience is that the +`oxicloud` binary depends on the SvelteKit build output (`static-dist/` +under `/static-dist/`, resolved by +`src/interfaces/web/mod.rs::resolve_static_path` at boot). Two files +to distribute per platform is friction; a single self-contained binary +is what users actually want. + +The ask has two parts: + +1. Ship **single-file binaries with frontend assets embedded**, for + the common Linux targets and macOS. +2. Audit the current binary set — the crate produces 6+ binaries today, + some of which are test-only. Strip anything that shouldn't ship to + end users. + +The intended outcome: a `v0.9.0` release attaches **4 musl-static +tarballs** (Linux amd64/arm64 + macOS Intel/Apple Silicon), each +~15-30 MB, containing a **single `oxicloud` binary** with assets + +operator tools + one-off migrations all baked in. User extracts the +tarball, sets `DATABASE_URL`, runs `./oxicloud` — server up. +Subcommands (`oxicloud opaque setup`, `oxicloud migrate +nfc-filenames --dry-run`) provide operator access to the same tools +currently split across `oxicloud-cli` and `migrate-nfc-filenames`. + +Design shape (confirmed 2026-08-27): + +- **musl-only Linux** — parity with the existing Docker image (Alpine + base), no glibc-version fragmentation +- **Assets embedded via `rust-embed` with compile-time deflate + compression** — smaller binary +- **`bundled-assets` is opt-in** — default `cargo build` unchanged; + `just dev` still uses the filesystem `ServeDir` with Vite HMR +- **Single unified binary** — `oxicloud`, `oxicloud-cli`, and + `migrate-nfc-filenames` collapse into one clap-driven executable + with implicit-server default (backwards compat with existing Docker + CMD / systemd units) + +## Current binary inventory + +From `Cargo.toml` + `src/bin/`: + +| Binary | Path | Purpose | Ship to end users? | +|---|---|---|---| +| `oxicloud` | `src/main.rs` (implicit) | Server | **YES** | +| `oxicloud-cli` | `src/bin/oxicloud-cli.rs` | Operator toolbox (`opaque setup/reset`) | **MERGED** — absorbed into `oxicloud` per Deliverable 1b | +| `migrate-nfc-filenames` | `src/bin/migrate-nfc-filenames.rs` | One-off filename migration (historical, June 2026 fix) | **MERGED** — absorbed into `oxicloud migrate nfc-filenames` per Deliverable 1a→1b | +| `generate-openapi` | `src/bin/generate-openapi.rs` | Regenerate `resources/gen/openapi.json` | NO — dev tool, gate behind `dev_tools` feature | +| `opaque-hurl-helper` | `src/bin/opaque-hurl-helper.rs` | Hurl test companion (OPRF client) | NO — gate behind `test_utils` feature | +| `dpop-hurl-helper` | `src/bin/dpop-hurl-helper.rs` | Hurl test companion (ES256 DPoP proof) | NO — gate behind `test_utils` feature | +| `load-seed` | `src/bin/load-seed.rs` | Test fixture seeder | Already gated behind `load_seed_bin` feature ✅ | + +After Deliverables 1 + 1a + 1b, `cargo build --release --bins` +produces exactly ONE binary: `oxicloud`. That single binary ships in +the tarball and in the Docker image. + +## Deliverables + +### 1. Squash test/dev binaries with `required-features` + +Cargo respects `required-features` per `[[bin]]` — a binary is only +built when its listed features are active. This gates test helpers +out of `cargo build --release --bins` cleanly without needing custom +Cargo commands or shell trimming. + +Edits to `Cargo.toml`: + +```toml +[features] +# ... existing features ... +dev_tools = [] # NEW: gates ops tooling that shouldn't ship + +[[bin]] +name = "opaque-hurl-helper" +path = "src/bin/opaque-hurl-helper.rs" +required-features = ["test_utils"] # NEW gate + +[[bin]] +name = "dpop-hurl-helper" +path = "src/bin/dpop-hurl-helper.rs" +required-features = ["test_utils"] # NEW gate + +[[bin]] +name = "generate-openapi" +path = "src/bin/generate-openapi.rs" +required-features = ["dev_tools"] # NEW gate — `just openapi` flips it + +# [[bin]] name = "migrate-nfc-filenames" ← DELETED per Deliverable 1a +# [[bin]] name = "oxicloud-cli" ← DELETED per Deliverable 1b +``` + +Existing invocations that need adjustment: + +- `just openapi` recipe → add `--features dev_tools` to the underlying + `cargo run --bin generate-openapi` call (currently `cargo run --bin + generate-openapi` per justfile) +- `tests/api/run.sh` → add `--features test_utils` when building the + two hurl helpers (shape confirmed: `cargo build [--release] --bin + opaque-hurl-helper` / same for dpop in each helper's build-if-missing + branch) + +After these edits + Deliverables 1a + 1b: `cargo build --release --bins` +produces exactly ONE binary — `oxicloud`. Everything else falls out of +the default build set. + +### 1a. Merge `migrate-nfc-filenames` into `oxicloud-cli` + +The standalone `migrate-nfc-filenames` binary is a June-2026 one-off: +it cleans up NFD/NFC filename collisions in databases populated +before the write-time fix (`normalize_storage_name()` at +`src/domain/services/path_service.rs:36`, called from +`src/infrastructure/repositories/pg/file_blob_read_repository.rs:1062`). +New installs never need it; only pre-June 2026 databases do. + +`oxicloud-cli`'s header docstring (`src/bin/oxicloud-cli.rs:20-23`) +already documents the growth pattern for absorbing tools like this: + +> *"each new domain gets its own module below (e.g. `mod opaque`) +> with a `#[derive(Subcommand)]` enum for its actions and a +> `run(args) -> ExitCode` entrypoint. Keep each module self-contained +> so a future extraction is a file move."* + +Note: this Deliverable is an intermediate step. Deliverable 1b then +absorbs `oxicloud-cli` itself into `oxicloud`, so the final CLI form +becomes `oxicloud migrate nfc-filenames --dry-run` — but 1a lands +first so the migration logic is proven inside the clap subcommand +tree before the main-binary merge. + +Edits: + +- **New `mod migrate` in `src/bin/oxicloud-cli.rs`** — moves the ~149 + non-boilerplate lines from `migrate-nfc-filenames.rs::main()` into + a `run_nfc_filenames(dry_run: bool) -> ExitCode` function. + `env::args()` parsing goes away; clap handles it. +- **Delete `src/bin/migrate-nfc-filenames.rs`**. +- **Delete the `[[bin]]` entry** in `Cargo.toml`. +- **Update `Dockerfile`** — 6 references to `migrate-nfc-filenames` + (build commands at :46, :49, :89, `cp` steps at :130, :143, doc + comment at :170, `COPY --chmod=755 --from=app` at :173). +- **Update `docs/plan/benchmake-and-performance-tracking.md`** — 2 + references to `migrate-nfc-filenames` at lines :44 and :161. Reword + to reference `oxicloud-cli migrate nfc-filenames` (or, after 1b, + `oxicloud migrate nfc-filenames`) and update the Cargo.toml + placement example. +- **Any operator runbook** that documents `docker exec + migrate-nfc-filenames --dry-run` becomes `docker exec + oxicloud-cli migrate nfc-filenames --dry-run` (intermediate) then + `docker exec oxicloud migrate nfc-filenames --dry-run` + after 1b. + +Effort: ~1.5 hours mechanical. Extracts the "should the tarball ship +migrate-nfc-filenames?" question entirely — everything now ships as +one operator toolbox binary that also happens to include the +historical migration. + +Future v1.0 removal path (deferred): delete `mod migrate` block + one +enum variant + docs. Much cleaner than removing a whole `.rs` file + +Cargo entry + Dockerfile refs. + +### 1b. Merge `oxicloud-cli` into `oxicloud` + +Single binary — server + operator tools + migrations — with an +**implicit-server** subcommand tree. `oxicloud` with no arguments +starts the server (backwards compat with existing Docker CMD / +systemd units / user configs). Subcommands add operator actions on +top. + +After merge, the CLI shape is: + +``` +$ oxicloud --help +Usage: oxicloud [OPTIONS] [COMMAND] + +Commands: + opaque OPAQUE aPAKE substrate management + migrate One-time data migrations + +If no command is given, oxicloud starts the server (see docs/config). +``` + +Concrete forms: + +- `oxicloud` — start server (unchanged) +- `oxicloud opaque setup` — was `oxicloud-cli opaque setup` +- `oxicloud opaque reset --user alice --dry-run` — was `oxicloud-cli + opaque reset ...` +- `oxicloud migrate nfc-filenames --dry-run` — was + `migrate-nfc-filenames --dry-run` (via Deliverable 1a) + +**Backwards-compat guarantee**: `oxicloud` with no args continues to +start the server. Every existing `CMD ["oxicloud"]`, `ExecStart=/usr/local/bin/oxicloud`, +docker-compose entry, and k8s Deployment keeps working unchanged. +Users updating to v0.9.0 see no surprise. + +**Migration impact**: the user-visible break is that `oxicloud-cli +opaque setup` (etc.) no longer exists as a separate binary. Given the +current audience for `oxicloud-cli` is very small (essentially only +the maintainer), the migration cost is trivial. Any user who had +scripted it can adapt with a one-line find/replace. + +Edits: + +- **`src/main.rs`** — top of `main()`, before the current server + init, parse args via clap. If a subcommand is provided, dispatch + to it and exit; otherwise fall through to the existing server-init + path. Zero-arg startup cost stays ≤ microseconds (clap parse of + empty args). +- **`src/cli/mod.rs`** — NEW module. Contains the `Domain` enum + the + `opaque` and `migrate` submodules moved from + `src/bin/oxicloud-cli.rs`. Each subcommand module keeps its + self-contained shape per the growth pattern documented in the + old `oxicloud-cli.rs` header. +- **Delete `src/bin/oxicloud-cli.rs`** entirely. +- **Delete the `[[bin]] name = "oxicloud-cli"` block** in `Cargo.toml`. +- **`Dockerfile`** — drop all 4 references to `oxicloud-cli` (build + target lines + COPY steps). Simplified build command becomes + `cargo build --release --bin oxicloud` — single-binary. +- **Docs** — all `docker exec oxicloud-cli + ` become `docker exec oxicloud + `. Same shape, one fewer word. + +Effort: ~2 hours mechanical. Comparable to Deliverable 1a but with +slightly more care at the `main.rs` entry point for the args-vs-server +branch. + +**Tarball layout simplification** — the tarball now ships exactly +ONE binary: + +``` +oxicloud-0.9.0-/ +├── oxicloud (single file, server + tools + embedded assets) +├── example.env +├── LICENSE +└── README-install.md +``` + +That's the "just download and run" ethos in physical form: one file, +one command, done. + +### 2. Add `bundled-assets` cargo feature + +Purpose: at compile time, choose between filesystem-served static +assets (current behaviour — filesystem `ServeDir`) and +embedded-into-binary assets (via `rust-embed`). Feature is +**opt-in** — the default `cargo build --release` still produces a +filesystem-based binary, matching the current Docker image behaviour +(where assets are separate volume layers). Release tarballs are built +with `--features bundled-assets`. + +**Dev mode is untouched.** `just dev` runs `PROFILE=dev cargo run` + +`npm run dev`, neither of which activates `bundled-assets`. The dev +workflow continues to: + +- Serve from `frontend/` via Vite's dev server with HMR +- Backend reads static assets from `/static-dist/` via the + usual `ServeDir` (or falls back to `frontend/static/` when the + build hasn't been run) +- No rebuild required to change locales, styles, or vendor JS + +The `bundled-assets` code paths only compile when the feature is +explicitly enabled — under a `#[cfg(feature = "bundled-assets")]` gate. +The non-feature build's binary shape, ergonomics, and dev loop stay +identical to today. + +Measured footprint (2026-08-27): + +| Slice | Size | Notes | +|---|---|---| +| Total `static-dist/` uncompressed | **9.8 MB** | 499 files | +| `_app/` (SvelteKit bundle) | 3.3 MB | JS + CSS chunks | +| `vendors/` | 3.6 MB | maplibre-gl 1.0 MB, pdf.worker 1.0 MB, others | +| `locales/` | 2.2 MB | 16 locales, ru.json + hi.json largest at ~116-140 KB | +| `logo/`, `geo/`, `basemaps/`, `workers/`, misc | ~600 KB | | +| **`.tar.gz` compressed** | **4.65 MB** | realistic embed cost after brotli/gzip inside binary | +| **`.tar.xz` compressed** | **4.22 MB** | not what rust-embed uses; reference only | + +Expected release-binary size with embed: `oxicloud` today ships in +the 30-60 MB range (stripped, LTO). Add ~5-10 MB for embedded +static-dist. Tarball compression on top → ~20-30 MB shipped per +platform. Four platforms × ~25 MB = ~100 MB per release. Well within +GitHub Releases limits. + +Cargo.toml additions: + +```toml +[features] +bundled-assets = ["dep:rust-embed", "dep:mime_guess"] + +[dependencies] +rust-embed = { version = "8", features = ["compression"], optional = true } +mime_guess = { version = "2", optional = true } +``` + +Runtime shape — a new module `src/interfaces/web/embedded.rs`: + +```rust +#[cfg(feature = "bundled-assets")] +#[derive(rust_embed::RustEmbed)] +#[folder = "static-dist/"] // ← repo-root, matches SvelteKit adapter-static output +#[include = "*"] +#[exclude = "*.br"] // Vite's precompressed sibling — response compression handles on wire +#[exclude = "*.gz"] // ditto +pub struct EmbeddedAssets; +``` + +The `#[folder]` path is relative to Cargo.toml (repo root), where the +SvelteKit adapter-static config in `frontend/svelte.config.js` emits: + +```js +adapter: adapter({ + pages: '../static-dist', + assets: '../static-dist', + ... +}) +``` + +The current filesystem shape (at `src/interfaces/web/mod.rs:47-106`) +is more than one `ServeDir` — the embed swap replaces FOUR sites, all +downstream of `resolve_static_path()`: + +1. **`spa` ServeDir** (`mod.rs:60-63`) — root fallback with + `precompressed_br().precompressed_gzip()` and SPA-shell fallback + pointing at `/index.html`. Under embed: an axum handler + that resolves the request path against `EmbeddedAssets::get()`, + 200 with correct MIME (via `mime_guess`) if hit, otherwise return + the embedded `index.html` bytes with `text/html` for SPA client-routing. +2. **`app_immutable` ServeDir** (`mod.rs:66-77`) — nested at + `/_app/immutable` with `Cache-Control: public, max-age=31536000, + immutable`. Under embed: same handler shape as (1), scoped to + the `_app/immutable/` prefix, plus a `.layer()` that stamps the + immutable cache header. +3. **`ServeFile::new(index.html)`** SPA fallback (`mod.rs:63`) — + folds into (1)'s not-found path. +4. **CSP inline-script scan** (`mod.rs:163-233`) — currently reads + every `.html` file in the resolved static dir via + `std::fs::read_dir` + `std::fs::read_to_string` at boot to compute + SHA-256 CSP source expressions for every inline `
@@ -593,10 +746,12 @@

- {#if hasBatch} + {#if batchJob} + {@const batch = batchJob} + + {#if batch.repair_description} + + {/if} {/if} {:else} - - {#if supportsDeep(job.name)} + + {@const hasRunVariants = supportsDeep(job.name) || supportsRepair(job)} + - {/if} + {#if hasRunVariants} + + {#if runMenuOpen[job.name]} + + {/if} + {/if} + {/if} - {#if isRunning(job) && canExpand} + + {#if isRunning(job) && isRecoverable(job)} {#if isRecoverable(job)} + {#if expandedJob === job.name && job.description} + + +

{job.description}

+ + + {/if} + + {#if expandedJob === job.name && isRecoverable(job)}
@@ -1304,6 +1590,90 @@ color: var(--color-danger-text-alt); } + /* Warn variant — used for actions that mutate data but are content- + safe / reversible-in-outcome (e.g. Repair ref_counts). Signals + "read the tooltip and the confirm before clicking" without the + danger red reserved for destructive delete-style buttons. */ + .jobs-panel__btn--warn { + border-color: var(--color-warning-border); + color: var(--color-warning-text); + } + + /* Split-button — inline flex holding a primary "Run" (fires default + action) and a chevron (opens the variants menu). `position: + relative` anchors the menu below the toggle. Only rendered on + rows whose job supports at least one variant; plain-Run rows + sidestep this whole structure. */ + .jobs-panel__split { + display: inline-flex; + position: relative; + } + + /* Attached-button trick: main loses its right border-radius, toggle + loses its left. Toggle also loses its left border so the two + don't render a double-thick divider. */ + .jobs-panel__split-main { + border-top-right-radius: 0; + border-bottom-right-radius: 0; + } + + .jobs-panel__split-toggle { + border-top-left-radius: 0; + border-bottom-left-radius: 0; + border-left: none; + padding-left: 0.35rem; + padding-right: 0.35rem; + } + + /* The variants menu — dropdown below the toggle, right-aligned so + it doesn't overflow the Actions column edge into the next row's + badge cell. Shadow + surface bg mirror the /files upload + dropdown (`upload-dropdown-menu`); using local CSS here rather + than the ported class so the jobs-panel keeps its scoped styling. */ + .jobs-panel__run-menu { + position: absolute; + top: calc(100% + 2px); + right: 0; + z-index: 30; + min-width: 10rem; + background: var(--color-bg-surface); + border: 1px solid var(--color-border); + border-radius: var(--radius-md, 6px); + box-shadow: var(--shadow-md); + padding: 0.25rem 0; + } + + .jobs-panel__run-menu-item { + display: flex; + align-items: center; + gap: 0.5rem; + width: 100%; + padding: 0.4rem 0.75rem; + background: transparent; + border: none; + text-align: left; + font: inherit; + color: var(--color-text); + cursor: pointer; + white-space: nowrap; + } + + .jobs-panel__run-menu-item:hover:not(:disabled) { + background: var(--color-bg-hover); + } + + .jobs-panel__run-menu-item:disabled { + opacity: 0.5; + cursor: not-allowed; + } + + /* Warn colour on the menu item mirrors the button variant so the + Repair option carries the same "attention-worthy but not + destructive" visual weight as its top-bar counterpart. */ + .jobs-panel__run-menu-item--warn { + color: var(--color-warning-text); + } + .jobs-panel__pill { display: inline-block; padding: 0.1rem 0.5rem; @@ -1342,6 +1712,34 @@ color: var(--color-text-muted); } + /* "read-only" sits beside the job name and answers the question an + operator asks before every trigger. Deliberately quiet — it marks + the safe case, so it should not compete with outcome pills. */ + .jobs-panel__pill--readonly { + margin-left: 0.4rem; + background: var(--color-bg-subtle); + color: var(--color-text-muted); + font-size: 0.72rem; + font-weight: 400; + vertical-align: middle; + } + + /* Opens the expanded block, so it carries the drawer's background and + drops its own separator — the runs table below it is part of the + same block, not a new entry. */ + .jobs-panel__desc-row td { + padding-left: 2rem; /* clears the chevron, lines up with the name */ + border-bottom-color: transparent; + background: var(--color-bg-subtle); + } + + .jobs-panel__description { + margin: 0; + color: var(--color-text-muted); + font-size: 0.8rem; + line-height: 1.4; + } + .jobs-panel__runs { background: var(--color-bg-subtle); } diff --git a/frontend/src/lib/components/AppShell.svelte b/frontend/src/lib/components/AppShell.svelte index 595b5637..74f21ad1 100644 --- a/frontend/src/lib/components/AppShell.svelte +++ b/frontend/src/lib/components/AppShell.svelte @@ -437,11 +437,11 @@ { mode: 'dark', icon: 'moon', label: t('user_menu.theme.dark', 'Dark') } ]; - const storagePct = $derived( - session.user && session.user.storage_quota_bytes > 0 - ? Math.min(100, (session.user.storage_used_bytes / session.user.storage_quota_bytes) * 100) - : 0 - ); + const storagePct = $derived.by(() => { + const full = session.me?.full; + if (!full || full.storage_quota_bytes <= 0) return 0; + return Math.min(100, (full.storage_used_bytes / full.storage_quota_bytes) * 100); + }); const initials = $derived(userInitials(session.user?.username || session.user?.email)); @@ -655,12 +655,12 @@
- {#if session.user.storage_quota_bytes > 0} - {Math.round(storagePct)}% · {formatBytes(session.user.storage_used_bytes)} / {formatBytes( - session.user.storage_quota_bytes + {#if (session.me?.full.storage_quota_bytes ?? 0) > 0} + {Math.round(storagePct)}% · {formatBytes(session.me?.full.storage_used_bytes ?? 0)} / {formatBytes( + session.me?.full.storage_quota_bytes ?? 0 )} {:else} - {formatBytes(session.user.storage_used_bytes)} + {formatBytes(session.me?.full.storage_used_bytes ?? 0)} {/if}
@@ -903,18 +903,18 @@
- {#if session.user.storage_quota_bytes > 0} + {#if (session.me?.full.storage_quota_bytes ?? 0) > 0} {t( 'storage.used', { percentage: Math.round(storagePct), - used: formatBytes(session.user.storage_used_bytes), - total: formatBytes(session.user.storage_quota_bytes) + used: formatBytes(session.me?.full.storage_used_bytes ?? 0), + total: formatBytes(session.me?.full.storage_quota_bytes ?? 0) }, '{{percentage}}% used ({{used}} / {{total}})' )} {:else} - {formatBytes(session.user.storage_used_bytes)} + {formatBytes(session.me?.full.storage_used_bytes ?? 0)} {/if}
diff --git a/frontend/src/lib/components/AppShell.test.ts b/frontend/src/lib/components/AppShell.test.ts index 1439ecff..04089694 100644 --- a/frontend/src/lib/components/AppShell.test.ts +++ b/frontend/src/lib/components/AppShell.test.ts @@ -26,16 +26,27 @@ const children = createRawSnippet(() => ({ beforeEach(() => { vi.clearAllMocks(); pageState.url = new URL('http://localhost/files'); - session.user = { - id: '1', - username: 'admin', - email: 'a@x.test', - given_name: 'A', - family_name: 'B', - role: 'admin', - storage_used_bytes: 10, - storage_quota_bytes: 100, - is_external: false + // Post the three-layer UserDto refactor, `session.user` is a + // derived accessor over `session.me.full.user`; only `session.me` + // is settable. Fixture composes the nested shape — public identity + // (username/email/name) on `.full.user`, admin+self extras + // (storage_*, has_password) on `.full`, self-only bag (ui_prefs, + // dpop_bound, force_password_change, can_edit_image) at the top. + // See docs/plan/userdto-refactor.md. + session.me = { + full: { + user: { + id: '1', + username: 'admin', + email: 'a@x.test', + given_name: 'A', + family_name: 'B', + role: 'admin', + is_external: false + }, + storage_used_bytes: 10, + storage_quota_bytes: 100 + } } as never; }); diff --git a/frontend/src/lib/components/EmptyState.svelte b/frontend/src/lib/components/EmptyState.svelte index 4403103f..1e0c606b 100644 --- a/frontend/src/lib/components/EmptyState.svelte +++ b/frontend/src/lib/components/EmptyState.svelte @@ -18,7 +18,19 @@ let { icon, title, hint, error = false, children }: Props = $props(); -
+ +
{#if icon}{/if} {#if title}

{title}

{/if} {#if hint}

{hint}

{/if} diff --git a/frontend/src/lib/components/UserVignette.svelte b/frontend/src/lib/components/UserVignette.svelte index a1c0fe12..57cd999e 100644 --- a/frontend/src/lib/components/UserVignette.svelte +++ b/frontend/src/lib/components/UserVignette.svelte @@ -33,6 +33,7 @@ const label = $derived(resolved?.name ?? fallbackLabel ?? userId); const email = $derived(resolved?.email || fallbackSublabel || ''); const isExternal = $derived(resolved?.isExternal ?? false); + const isOnline = $derived(resolved?.isOnline ?? false); const image = $derived(resolved?.image ?? null); const colorIndex = $derived(avatarColorIndex(userId)); const initials = $derived(userInitials(label)); @@ -50,6 +51,22 @@ {/if} + {#if isOnline} + + + {/if} {label} @@ -128,6 +145,27 @@ font-size: 9px; } + /* Presence dot — top-right, symmetric with `.uv__badge` at + bottom-right so the two corners don't collide. Slightly smaller + (10x10 vs the badge's 16x16) because it's a pure signal — no + icon, no text. The 2px `--color-bg-surface` border creates a + visual gap between dot and avatar so the green pops out cleanly + regardless of avatar palette (photo, dark initials, light + initials). `box-sizing: border-box` keeps the inner circle's + green footprint at 6x6 — same visual weight the sessions-table + dot has. See `docs/plan/sessions.md` § UI. */ + .uv__presence { + position: absolute; + right: -2px; + top: -2px; + width: 10px; + height: 10px; + border-radius: 50%; + background: var(--color-success-alt); + border: 2px solid var(--color-bg-surface); + box-sizing: border-box; + } + .uv__text { display: flex; flex-direction: column; diff --git a/frontend/src/lib/stores/preferences.svelte.ts b/frontend/src/lib/stores/preferences.svelte.ts index e9e141d0..a905da65 100644 --- a/frontend/src/lib/stores/preferences.svelte.ts +++ b/frontend/src/lib/stores/preferences.svelte.ts @@ -69,12 +69,15 @@ const PATCH_DEBOUNCE_MS = 500; class PreferencesStore { /** - * The typed view of the bag. Derived from `session.user?.ui_preferences` - * so signing in / out / refresh flips it in lockstep with the session. + * The typed view of the bag. Derived from `session.me?.ui_preferences` + * (moved from public `User.ui_preferences` to `SelfUser.ui_preferences` + * as part of the three-layer UserDto refactor — the bag is self-only + * state, not something other authenticated callers should see). + * Signing in / out / refresh flips it in lockstep with the session. * Reads pass through DEFAULTS for any missing key. */ private bag = $derived>( - (session.user?.ui_preferences as Record | undefined) ?? {} + (session.me?.ui_preferences as Record | undefined) ?? {} ); // ── Typed accessors ────────────────────────────────────────── @@ -100,11 +103,14 @@ class PreferencesStore { * `jsonb_strip_nulls` after the merge). */ set(patch: Partial>): void { - if (!session.user) return; + if (!session.me) return; - // Optimistic local write — mutate the reactive user shallowly. + // Optimistic local write — mutate the reactive me shallowly. + // `ui_preferences` lives on `SelfUser` (self-only), not on the + // public `User` slice, so the mutation stays at the SelfUser + // level. The nested `full` / `full.user` blocks are untouched. const nextBag = { - ...((session.user.ui_preferences as Record | undefined) ?? {}), + ...((session.me.ui_preferences as Record | undefined) ?? {}), ...patch }; // Strip any explicit-null locally so the derived getters see the @@ -114,7 +120,7 @@ class PreferencesStore { for (const [k, v] of Object.entries(patch)) { if (v === null) delete (nextBag as Record)[k]; } - session.user = { ...session.user, ui_preferences: nextBag }; + session.me = { ...session.me, ui_preferences: nextBag }; // Accumulate keys so successive `set` calls before the debounce // fires collapse into a single PATCH body — matters for @@ -131,16 +137,20 @@ class PreferencesStore { this.pendingPatch = {}; if (Object.keys(patch).length === 0) return; - const previousUser = session.user; + // `session.user` is a derived read-through on `session.me.full.user` + // — the source of truth is `session.me: SelfUser`. Snapshot + assign + // there so the optimistic update / rollback matches the store shape + // (see `docs/plan/userdto-refactor.md` for the layering). + const previousMe = session.me; try { const updated = await updateProfile({ ui_preferences: patch }); - session.user = updated; + session.me = updated; } catch { // Roll back to whatever the server last confirmed. The // optimistic local mutation is discarded and the derived // `hideDotfiles` / other getters snap back on the next // reactivity tick. - session.user = previousUser; + session.me = previousMe; ui.notify( t('preferences.save_failed', "Couldn't save your preference. Please try again."), 'error' diff --git a/frontend/src/lib/stores/session.svelte.test.ts b/frontend/src/lib/stores/session.svelte.test.ts index 20325726..85450069 100644 --- a/frontend/src/lib/stores/session.svelte.test.ts +++ b/frontend/src/lib/stores/session.svelte.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { User } from '$lib/api/types'; +import type { SelfUser } from '$lib/api/types'; // `vi.mock` is hoisted above imports, so the spy it references must be created // with `vi.hoisted` (a plain top-level const isn't initialised yet when the @@ -14,7 +14,14 @@ vi.mock('$lib/api/endpoints/auth', () => ({ import { session } from './session.svelte'; -const userWithUsage = (used: number) => ({ storage_used_bytes: used }) as unknown as User; +// `storage_used_bytes` moved to `FullUser` (embedded inside `SelfUser`) +// as part of the three-layer UserDto refactor +// (`docs/plan/userdto-refactor.md`). Build a minimal SelfUser shape that +// satisfies the type checker without hand-populating every field the +// production shape carries — the test only cares about the usage read +// path (`session.me.full.storage_used_bytes`). +const userWithUsage = (used: number) => + ({ full: { storage_used_bytes: used } }) as unknown as SelfUser; describe('session.refresh', () => { beforeEach(() => { @@ -25,7 +32,7 @@ describe('session.refresh', () => { it('pulls the fresh storage usage into the reactive user (upload/delete sync)', async () => { fetchMeMock.mockResolvedValue(userWithUsage(2048)); await session.refresh(); - expect(session.user?.storage_used_bytes).toBe(2048); + expect(session.me?.full.storage_used_bytes).toBe(2048); }); it('leaves the current user intact when the probe returns null', async () => { @@ -33,7 +40,7 @@ describe('session.refresh', () => { await session.refresh(); fetchMeMock.mockResolvedValue(null); await session.refresh(); - expect(session.user?.storage_used_bytes).toBe(2048); + expect(session.me?.full.storage_used_bytes).toBe(2048); }); it('leaves the current user intact when the probe throws', async () => { @@ -41,6 +48,6 @@ describe('session.refresh', () => { await session.refresh(); fetchMeMock.mockRejectedValue(new Error('network')); await session.refresh(); - expect(session.user?.storage_used_bytes).toBe(2048); + expect(session.me?.full.storage_used_bytes).toBe(2048); }); }); diff --git a/frontend/src/lib/stores/session.svelte.ts b/frontend/src/lib/stores/session.svelte.ts index bc2022c1..a46e1e4d 100644 --- a/frontend/src/lib/stores/session.svelte.ts +++ b/frontend/src/lib/stores/session.svelte.ts @@ -11,17 +11,39 @@ import { setLogoutInProgress } from '$lib/api/client'; import { hasSessionHint } from '$lib/api/csrf'; import { seedNonceFromCookie } from '$lib/auth/dpop-proof'; import { drives } from '$lib/stores/drives.svelte'; -import type { User } from '$lib/api/types'; +import type { PublicUser, SelfUser } from '$lib/api/types'; import { ensureActiveUser } from '$lib/utils/localStoragePrefs'; +/** + * Session store — the authenticated user and derived flags. + * + * Post the three-layer UserDto refactor (`docs/plan/userdto-refactor.md`), + * `/api/auth/me` returns `SelfUser` (composed: + * `SelfUser.full.user: PublicUser`). Two shorthand accessors keep every + * existing consumer readable: + * + * - `session.user` → `PublicUser` (via `me.full.user`). Every callsite + * that read `session.user.username / email / id / role / image / + * is_external / given_name / family_name / is_online` keeps working. + * - `session.me` → full `SelfUser`. New code that needs self-only or + * admin-visible fields (`has_password`, `is_dpop_bound`, `active`, + * `ui_preferences`, `federation_kind`, `last_login_at`, quotas, …) + * reads through `session.me.full.foo` or `session.me.foo`. + */ class SessionStore { - user = $state(null); + /** Full `/api/auth/me` payload. Null when unauthenticated. */ + me = $state(null); loaded = $state(false); homeFolderId = $state(null); homeFolderName = $state(null); - isExternalUser = $derived(this.user?.is_external ?? false); - isAuthenticated = $derived(this.user !== null); + /** Public-identity shorthand — same fields any authenticated caller + * can see. Every legacy `session.user.foo` read (username, email, id, + * role, image, is_external, given_name, family_name, is_online) still + * works via this derived accessor. */ + user = $derived(this.me?.full.user ?? null); + isExternalUser = $derived(this.me?.full.user.is_external ?? false); + isAuthenticated = $derived(this.me !== null); /** * TRUE when the backend has set `force_password_change_at_next_login` * on this account — an admin picked a temporary password and the @@ -33,7 +55,7 @@ class SessionStore { * flag (or a malformed `/me` response) doesn't accidentally * quarantine every user. */ - mustChangePassword = $derived(this.user?.force_password_change === true); + mustChangePassword = $derived(this.me?.force_password_change === true); /** * Resolve the session once. Probes /api/auth/me; on 401 it makes a single @@ -41,15 +63,15 @@ class SessionStore { * what to do with an unauthenticated result. Idempotent: subsequent calls * return the cached result (so client-side navigation doesn't re-probe). */ - async load(): Promise { - if (this.loaded) return this.user; + async load(): Promise { + if (this.loaded) return this.me; // No JS-visible session hint ⇒ nothing to probe. The server sets // `oxicloud_csrf` alongside the HttpOnly session cookies and clears // it on logout, so a missing hint means no session. Skips the // doomed 2× /me + /refresh burst that would otherwise fire on // every first landing / post-logout re-mount with no cookies. if (!hasSessionHint()) { - this.user = null; + this.me = null; this.loaded = true; return null; } @@ -71,25 +93,25 @@ class SessionStore { // otherwise clutter the audit stream. Fire-and-forget // so a slow IndexedDB open doesn't stall app boot. if (me.is_dpop_bound === false) void bindDpopIfPossible(); - } else this.user = null; + } else this.me = null; } catch { - this.user = null; + this.me = null; } this.loaded = true; - return this.user; + return this.me; } /** * Set the authenticated user AND run per-user localStorage cleanup * (see `$lib/utils/localStoragePrefs::ensureActiveUser`). Direct - * `session.user = …` assignments skip the cleanup — always call + * `session.me = …` assignments skip the cleanup — always call * `setUser` on login-flow entry points (form login, OIDC exchange, * existing-session probe) so a switch-account flow inside the same * tab observes the wipe. */ - setUser(user: User): void { - this.user = user; - ensureActiveUser(user.id); + setUser(me: SelfUser): void { + this.me = me; + ensureActiveUser(me.full.user.id); // Any successful login clears the session-teardown gate. Without // this, a logout → login within the same SPA session leaves the // gate stuck at `true` — the login POST is exempted via @@ -115,7 +137,7 @@ class SessionStore { async refresh(): Promise { try { const me = await fetchMe(); - if (me) this.user = me; + if (me) this.me = me; } catch { /* keep the existing user on a transient /api/auth/me failure */ } @@ -142,7 +164,7 @@ class SessionStore { } reset(): void { - this.user = null; + this.me = null; this.homeFolderId = null; this.homeFolderName = null; // Mark the store as `loaded` so any subsequent `session.load()` — diff --git a/frontend/src/routes/admin/[[tab]]/+page.svelte b/frontend/src/routes/admin/[[tab]]/+page.svelte index 037e4636..919a3770 100644 --- a/frontend/src/routes/admin/[[tab]]/+page.svelte +++ b/frontend/src/routes/admin/[[tab]]/+page.svelte @@ -63,6 +63,7 @@ type StorageTestResult } from '$lib/api/endpoints/admin'; import { createDrive, updateDrivePolicies } from '$lib/api/endpoints/drives'; + import { seedUser } from '$lib/api/endpoints/users'; import { ensureResolvers, resolveRecipient, @@ -70,7 +71,7 @@ type Recipient } from '$lib/api/endpoints/recipients'; import type { - AdminUserSummary, + FullUser, Drive, DriveMember, DrivePolicies, @@ -156,11 +157,11 @@ deleteUserModal !== null && deleteUserEmailInput.trim().toLowerCase() === deleteUserModal.email.toLowerCase() ); - function openDeleteUser(u: AdminUserSummary) { + function openDeleteUser(u: FullUser) { deleteUserModal = { - userId: u.id, - username: u.username || u.email, - email: u.email + userId: u.user.id, + username: u.user.username || u.user.email, + email: u.user.email }; deleteUserEmailInput = ''; } @@ -817,7 +818,7 @@ } // Users - let users = $state([]); + let users = $state([]); let total = $state(0); let pageIndex = $state(0); let usersError = $state(null); @@ -923,6 +924,12 @@ const page = await listUsers(PAGE_SIZE, pageIndex * PAGE_SIZE); users = page.users; total = page.total; + // Seed the per-user resolver cache with the row's `PublicUser` + // slice so every `UserVignette` mounted per row hits the cache + // synchronously — no per-row `/api/users/{id}` follow-up. + // Kills the N+1 that motivated widening `/api/admin/users` to + // carry the avatar (docs/plan/userdto-refactor.md § N+1). + for (const row of page.users) seedUser(row.user); } catch (e) { usersError = errorMessage(e); } @@ -1003,49 +1010,49 @@ } /** True for the signed-in admin's own row — guards self-destructive actions. */ - function isSelf(u: AdminUserSummary): boolean { - return u.id === currentAdminId; + function isSelf(u: FullUser): boolean { + return u.user.id === currentAdminId; } /** OIDC/SSO-provisioned account (no local password to reset). */ - function isOidcUser(u: AdminUserSummary): boolean { + function isOidcUser(u: FullUser): boolean { return u.federation_kind === 'oidc'; } /** Used-quota percentage (0 when unlimited) for the per-user progress bar. */ - function quotaPct(u: AdminUserSummary): number { + function quotaPct(u: FullUser): number { return u.storage_quota_bytes > 0 ? (u.storage_used_bytes / u.storage_quota_bytes) * 100 : 0; } - async function toggleRole(u: AdminUserSummary) { + async function toggleRole(u: FullUser) { if (isSelf(u)) return; - const role = u.role === 'admin' ? 'user' : 'admin'; + const role = u.user.role === 'admin' ? 'user' : 'admin'; if (!(await showConfirm(t('admin.confirm_role', { role }, 'Change role to {{role}}?')))) return; try { - await setUserRole(u.id, role); + await setUserRole(u.user.id, role); await loadUsers(); } catch (e) { reportError(e); } } - async function toggleActive(u: AdminUserSummary) { + async function toggleActive(u: FullUser) { if (isSelf(u) && u.active) return; const msg = u.active ? t('admin.confirm_deactivate', 'Deactivate this user?') : t('admin.confirm_activate', 'Activate this user?'); if (!(await showConfirm(msg))) return; try { - await setUserActive(u.id, !u.active); + await setUserActive(u.user.id, !u.active); await loadUsers(); } catch (e) { reportError(e); } } - function openQuota(u: AdminUserSummary) { + function openQuota(u: FullUser) { quotaModalError = null; quotaModal = { - userId: u.id, - username: u.username || u.email, + userId: u.user.id, + username: u.user.username || u.user.email, initialBytes: u.storage_quota_bytes }; } @@ -1069,8 +1076,8 @@ } } - function openReset(u: AdminUserSummary) { - resetModal = { userId: u.id, username: u.username || u.email }; + function openReset(u: FullUser) { + resetModal = { userId: u.user.id, username: u.user.username || u.user.email }; resetPassword = ''; resetError = null; } @@ -1094,7 +1101,7 @@ } } - function removeUser(u: AdminUserSummary) { + function removeUser(u: FullUser) { if (isSelf(u)) return; openDeleteUser(u); } @@ -1103,20 +1110,20 @@ // provisions a home drive + flips the is_external flag; irreversible // via the admin UI (there's no demote endpoint on purpose). Backend // refuses when magic-link login is disabled — surfaced as a toast. - async function promoteExternal(u: AdminUserSummary) { - if (!u.is_external) return; + async function promoteExternal(u: FullUser) { + if (!u.user.is_external) return; if ( !(await showConfirm( t( 'admin.confirm_promote_user', - { name: u.username || u.email }, + { name: u.user.username || u.user.email }, 'Promote {{name}} to an internal user? This provisions a home drive and gives the account a normal storage envelope. The account keeps its identity; magic-link login stays the way in unless a password is set later.' ) )) ) return; try { - await promoteUserToInternal(u.id); + await promoteUserToInternal(u.user.id); await loadUsers(); } catch (e) { reportError(e); @@ -1240,8 +1247,13 @@ .map(async (d) => { const ownerMember = nextMembers[d.id]?.find((m) => m.subject.type === 'user'); if (!ownerMember) return; - const user = await getUserAdmin(ownerMember.subject.id); - if (user) nextOwners[d.id] = user; + // `getUserAdmin` returns `FullUser` (admin-visible extras + // + nested `.user: PublicUser`). The drive row only reads + // public-identity fields (username, email, image) so keep + // the map typed as `PublicUser` and unwrap the embedded + // public block on insert. See docs/plan/userdto-refactor.md. + const full = await getUserAdmin(ownerMember.subject.id); + if (full) nextOwners[d.id] = full.user; }) ); personalDriveOwners = nextOwners; @@ -1723,6 +1735,16 @@ {:else if !dashboard}

{t('common.loading', 'Loading…')}

{:else} + + + +

{t('admin.section_accounts', 'User accounts')}

{dashboard.total_users}{t('admin.total_users', 'Total users')} @@ -1733,11 +1755,66 @@
{dashboard.admin_users}{t('admin.admin_users', 'Admins')}
-
- v{dashboard.server_version}{t('admin.version', 'Version')} +
+ {dashboard.external_users}{t( + 'admin.external_users', + 'External' + )}
+ +

+ {t('admin.section_activity', 'Live activity')} + + {t('admin.live', 'live')} + +

+
+
+ + + {dashboard.online_users} + + {t('admin.online_users', 'Online users')} +
+
+ + + {dashboard.online_sessions} + + {t('admin.online_sessions', 'Online sessions')} +
+
+ + +

{t('admin.section_system', 'System')}

@@ -1761,6 +1838,9 @@ {t('admin.quotas', 'Quotas')}
+
+ v{dashboard.server_version}{t('admin.version', 'Version')} +
{#if dashboard.users_over_quota > 0} @@ -1802,13 +1882,17 @@ row.kind === 'personal' ? t('admin.quota_personal', 'Personal drives') : t('admin.quota_shared', 'Shared drives')} + {@const total = row.unlimited_count + row.capped_count} {@const pct = row.capped_quota_bytes && row.capped_quota_bytes > 0 ? (row.used_bytes / row.capped_quota_bytes) * 100 : null} {#if row.capped_count > 0 || row.unlimited_count > 0} - {label} + + {total} + {label} + {#if row.capped_quota_bytes !== null && pct !== null} {formatBytes(row.used_bytes)} / {formatBytes(row.capped_quota_bytes)} @@ -2680,15 +2764,15 @@ - {#each users as u (u.id)} + {#each users as u (u.user.id)} {@const pct = quotaPct(u)}
{#if isSelf(u)} {t('admin.you_badge', 'you')} @@ -2703,11 +2787,11 @@ badge is `white-space: nowrap` so the badge label itself never wraps mid-word either. -->
- - {#if u.role === 'admin'}{/if} - {u.role} + + {#if u.user.role === 'admin'}{/if} + {u.user.role} - {#if u.is_external} + {#if u.user.is_external} {#if isOidcUser(u)} - - {u.federation_issuer} + + oidc {/if} {#if u.has_password} @@ -2825,7 +2909,7 @@ - {#if u.is_external} + {#if u.user.is_external}
- {#if u.is_external} + {#if u.user.is_external} {:else} @@ -2926,7 +3010,7 @@
-
{timeAgo(session.user.last_login_at)}
+
{timeAgo(session.me?.full.last_login_at)}
@@ -752,20 +762,20 @@

{t('profile.storage', 'Storage')}

-
{formatBytes(session.user.storage_used_bytes)}
+
{formatBytes(session.me?.full.storage_used_bytes ?? 0)}
{t('profile.used', 'Used')}
- {session.user.storage_quota_bytes > 0 - ? formatBytes(session.user.storage_quota_bytes) + {(session.me?.full.storage_quota_bytes ?? 0) > 0 + ? formatBytes(session.me?.full.storage_quota_bytes ?? 0) : '∞'}
{t('profile.quota', 'Quota')}
- {session.user.storage_quota_bytes > 0 ? `${storagePct}%` : '—'} + {(session.me?.full.storage_quota_bytes ?? 0) > 0 ? `${storagePct}%` : '—'}
{t('profile.usage', 'Usage')}
diff --git a/frontend/src/routes/profile/page.test.ts b/frontend/src/routes/profile/page.test.ts index 204a7744..10a2cf9b 100644 --- a/frontend/src/routes/profile/page.test.ts +++ b/frontend/src/routes/profile/page.test.ts @@ -1,10 +1,15 @@ import { it, expect, vi, beforeEach } from 'vitest'; import { render, screen, fireEvent, waitFor } from '@testing-library/svelte'; -const { session, ui } = vi.hoisted(() => ({ - session: { - loaded: true, - load: vi.fn(), +// Test-double session store. Post the three-layer UserDto refactor +// (docs/plan/userdto-refactor.md), production `session.user` is a +// derived accessor over `session.me.full.user`. The stub here mirrors +// that shape: `me` carries the whole SelfUser tree, and `user` mirrors +// `me.full.user` so any legacy `session.user.foo` read on the tested +// page keeps working through the mock without reproducing the derived +// mechanism. +const buildSelfMe = () => ({ + full: { user: { id: '1', username: 'admin', @@ -12,14 +17,41 @@ const { session, ui } = vi.hoisted(() => ({ given_name: 'A', family_name: 'B', role: 'admin', + is_external: false + }, + storage_used_bytes: 100, + storage_quota_bytes: 1000, + has_password: true + } +}); + +const { session, ui } = vi.hoisted(() => { + const me = { + full: { + user: { + id: '1', + username: 'admin', + email: 'a@x.test', + given_name: 'A', + family_name: 'B', + role: 'admin', + is_external: false + }, storage_used_bytes: 100, storage_quota_bytes: 1000, - is_external: false, has_password: true } - }, - ui: { notify: vi.fn() } -})); + }; + return { + session: { + loaded: true, + load: vi.fn(), + me, + user: me.full.user + }, + ui: { notify: vi.fn() } + }; +}); vi.mock('$lib/stores/session.svelte', () => ({ session })); vi.mock('$lib/stores/ui.svelte', () => ({ ui })); vi.mock('$lib/stores/dialogs.svelte', () => ({ confirmDialog: vi.fn() })); @@ -44,20 +76,13 @@ const m = (fn: unknown) => fn as ReturnType; beforeEach(() => { vi.clearAllMocks(); - // Reset the shared session each test (handlers may mutate session.user). + // Reset the shared session each test (handlers may mutate session.me + // on save / refresh). `me` is the SelfUser tree; `user` mirrors + // `me.full.user` for legacy `session.user.foo` reads. session.loaded = true; - session.user = { - id: '1', - username: 'admin', - email: 'a@x.test', - given_name: 'A', - family_name: 'B', - role: 'admin', - storage_used_bytes: 100, - storage_quota_bytes: 1000, - is_external: false, - has_password: true - }; + const me = buildSelfMe(); + session.me = me; + session.user = me.full.user; m(profile.listAppPasswords).mockResolvedValue([]); m(profile.updateProfile).mockResolvedValue(undefined); m(getOidcProviders).mockResolvedValue({ password_login_enabled: true }); diff --git a/frontend/static/locales/ar.json b/frontend/static/locales/ar.json index ec5d8a6f..a6ec4f3c 100644 --- a/frontend/static/locales/ar.json +++ b/frontend/static/locales/ar.json @@ -1201,11 +1201,11 @@ "progress_scanned_only_tooltip": "لا يوجد إجمالي متاح لهذا التشغيل (نشر شريط التقدم المسبق أو عدم قيام المستأجر بالإبلاغ عن موضوع قابل للعد).", "findings_present_tooltip": "قم بتوسيع هذا التشغيل لرؤية تفاصيل كل نتيجة.", "col_error": "خطأ", - "col_kind": "عطوف", + "col_kind": "نوع", "col_severity": "خطورة", "col_resource": "الموارد", "col_detail": "التفاصيل", - "run": "يجري", + "run": "تشغيل", "cancel": "يلغي", "refresh": "ينعش", "runs_title": "أشواط الأخيرة", @@ -1218,7 +1218,7 @@ "every_min": "كل دقيقة", "every_sec": "كل ق", "outcome_ok": "نعم", - "outcome_err": "يخطئ", + "outcome_err": "خطأ", "outcome_issues": "مشاكل", "outcome_notices": "إشعارات", "n_findings": "‹النتائج", diff --git a/frontend/static/locales/de.json b/frontend/static/locales/de.json index 38bd8b1a..2d5f1fd6 100644 --- a/frontend/static/locales/de.json +++ b/frontend/static/locales/de.json @@ -1182,8 +1182,8 @@ "gen_key": "Schlüssel generieren", "gen_key_warning": "Bewahren Sie diesen Schlüssel sicher auf. Bei Verlust sind die verschlüsselten Daten unwiederbringlich verloren.", "jobs": { - "run_all_consistency": "Führen Sie alle Konsistenzprüfungen durch", - "run_deep": "Lauf tief", + "run_all_consistency": "Alle Konsistenzprüfungen ausführen", + "run_deep": "Tiefenprüfung", "run_deep_hint": "Läuft auch langsame Varianten (Blob-Re-Hash, Bitrot-Erkennung).", "col_name": "Name", "col_cadence": "Kadenz", @@ -1205,7 +1205,7 @@ "col_severity": "Schwere", "col_resource": "Ressource", "col_detail": "Detail", - "run": "Laufen", + "run": "Ausführen", "cancel": "Stornieren", "refresh": "Aktualisieren", "runs_title": "Aktuelle Läufe", @@ -1218,7 +1218,7 @@ "every_min": "alle {{n}} Min", "every_sec": "alle {{n}} s", "outcome_ok": "OK", - "outcome_err": "ähm", + "outcome_err": "err", "outcome_issues": "Probleme", "outcome_notices": "Hinweise", "n_findings": "{{n}} Erkenntnisse", diff --git a/frontend/static/locales/en.json b/frontend/static/locales/en.json index 304e0b2e..cac7c577 100644 --- a/frontend/static/locales/en.json +++ b/frontend/static/locales/en.json @@ -546,6 +546,7 @@ "empty_hidden_hint": "Files whose name starts with '.' are hidden. Toggle the setting to see them.", "show_hidden": "Show hidden files", "upload_dotfile_hidden": "{{n}} file(s) uploaded but hidden by your dotfile preference.", + "upload_folder_not_ready": "Folder is still loading — please try again in a moment.", "rename_dotfile_hidden": "Renamed to '{{name}}' — now hidden by your preference.", "new_folder_dotfile_hidden": "Created folder '{{name}}' — hidden by your dotfile preference.", "dotfiles_hidden_toast": "Dotfiles hidden", @@ -836,6 +837,17 @@ "total_users": "Total Users", "active_users": "Active Users", "admins": "Admins", + "external_users": "External", + "external_users_tooltip": "Grant-only accounts — magic-link, OIDC-only, OCM recipients", + "online_users": "Online users", + "online_users_tooltip": "Distinct users with a session active in the last 5 minutes", + "online_sessions": "Online sessions", + "online_sessions_tooltip": "Non-revoked sessions active in the last 5 minutes — multi-device users contribute more than one", + "section_accounts": "User accounts", + "section_activity": "Live activity", + "section_system": "System", + "live": "live", + "live_tooltip": "Reflects sessions active in the last 5 minutes", "version": "Version", "storage_overview": "Storage Overview", "used": "Used", @@ -1136,6 +1148,10 @@ "revoked": "revoked", "expired": "expired", "active": "active", + "online": "online", + "idle": "idle", + "presence_online_tooltip": "Online — last seen {{ago}}", + "presence_idle_tooltip": "Idle — last seen {{ago}}", "revoke": "Revoke", "empty": "No sessions match the current filter.", "revoke_self_confirm": "⚠️ This is YOUR current session. Revoking it will log YOU out immediately and you'll have to sign back in. Continue?", @@ -1259,6 +1275,20 @@ "run_all_consistency": "Run all consistency checks", "run_deep": "Run deep", "run_deep_hint": "Also runs slow variants (blob re-hash, bitrot detection).", + "run_repair": "Repair ref_counts", + "run_repair_hint": "Corrects any drifted ref_counts (blobs + manifests) found by the audit. Content-safe — only counters change, not data.", + "run_repair_confirm_title_scoped": "Run {{name}} in repair mode?", + "run_mutating_confirm_title": "Run {{name}}?", + "run_mutating_confirm_body": "This job changes stored state when it runs.", + "mutates_never": "read-only", + "mutates_on_repair_only": "read-only unless repaired", + "startup": "at boot", + "startup_repair": "at boot · repair", + "startup_tooltip": "Configured in OXICLOUD_STARTUP_JOBS to run at every boot.", + "startup_repair_tooltip": "Configured in OXICLOUD_STARTUP_JOBS to run in repair mode at every boot.", + "run_variants_menu": "Run variants menu", + "run_repair_confirm": "Repair", + "triggered_ok_repair": "{{name}}: {{n}} counter(s) repaired", "col_name": "Name", "col_cadence": "Cadence", "col_last_run": "Last run", diff --git a/frontend/static/locales/es.json b/frontend/static/locales/es.json index 5f157a0b..881ba985 100644 --- a/frontend/static/locales/es.json +++ b/frontend/static/locales/es.json @@ -1183,15 +1183,15 @@ "time_just_now": "En este momento", "unchanged": "Déjelo en blanco para mantenerse actualizado", "jobs": { - "run_all_consistency": "Ejecute todas las comprobaciones de coherencia", - "run_deep": "Corre profundo", + "run_all_consistency": "Ejecutar todas las comprobaciones de coherencia", + "run_deep": "Análisis en profundidad", "run_deep_hint": "También ejecuta variantes lentas (repetición de blobs, detección de bitrot).", "col_name": "Nombre", "col_cadence": "Cadencia", "col_last_run": "última ejecución", "col_outcome": "Resultado", "col_state": "Estado", - "col_actions": "Comportamiento", + "col_actions": "Acciones", "col_started_at": "Comenzó", "col_status": "Estado", "col_duration": "Duración", @@ -1202,24 +1202,24 @@ "progress_scanned_only_tooltip": "No hay un total disponible para esta ejecución (implementación previa a la barra de progreso o el inquilino no informa un asunto contable).", "findings_present_tooltip": "Amplíe esta ejecución para ver detalles por hallazgo.", "col_error": "Error", - "col_kind": "Amable", + "col_kind": "Tipo", "col_severity": "Gravedad", "col_resource": "Recurso", "col_detail": "Detalle", - "run": "Correr", + "run": "Ejecutar", "cancel": "Cancelar", "refresh": "Refrescar", "runs_title": "Ejecuciones recientes", "run_json": "Resumen de ejecución (JSON)", "findings_title": "Recomendaciones", - "no_runs": "Aún no hay carreras.", + "no_runs": "Aún no hay ejecuciones.", "no_findings": "No hay resultados: ejecución limpia.", "on_demand": "Bajo demanda", "every_h": "cada {{n}} horas", "every_min": "cada {{n}} minutos", "every_sec": "cada {{n}} s", "outcome_ok": "OK", - "outcome_err": "errar", + "outcome_err": "err", "outcome_issues": "asuntos", "outcome_notices": "avisos", "n_findings": "{{n}} hallazgos", diff --git a/frontend/static/locales/fa.json b/frontend/static/locales/fa.json index 03a60e3f..73f1e92a 100644 --- a/frontend/static/locales/fa.json +++ b/frontend/static/locales/fa.json @@ -1165,7 +1165,7 @@ "gen_key_warning": "این کلید را به صورت ایمن ذخیره کنید. اگر از بین برود، داده های رمزگذاری شده به طور غیرقابل جبرانی از بین می روند.", "jobs": { "run_all_consistency": "تمام بررسی های سازگاری را اجرا کنید", - "run_deep": "عمیق بدو", + "run_deep": "بررسی عمیق", "run_deep_hint": "همچنین انواع آهسته را اجرا می کند (هش مجدد حباب، تشخیص بیتوت).", "col_name": "نام", "col_cadence": "آهنگ", @@ -1182,7 +1182,7 @@ "progress_scanned_only_tooltip": "مجموع برای این اجرا موجود نیست (پیش از پیشرفت نوار مستقر شده یا مستاجر موضوع قابل شمارش را گزارش نمی کند).", "findings_present_tooltip": "این اجرا را گسترش دهید تا جزئیات هر یافته را ببینید.", "col_error": "خطا", - "col_kind": "مهربان", + "col_kind": "نوع", "col_severity": "شدت", "col_resource": "منبع", "col_detail": "جزئیات", @@ -1199,7 +1199,7 @@ "every_min": "هر {{n}} دقیقه", "every_sec": "هر {{n}} ثانیه", "outcome_ok": "باشه", - "outcome_err": "اشتباه کن", + "outcome_err": "خطا", "outcome_issues": "مسائل", "outcome_notices": "اطلاعیه ها", "n_findings": "{{n}} یافته ها", diff --git a/frontend/static/locales/fr.json b/frontend/static/locales/fr.json index 14956565..0d6cfd6a 100644 --- a/frontend/static/locales/fr.json +++ b/frontend/static/locales/fr.json @@ -459,6 +459,7 @@ "empty_hidden_hint": "Les fichiers dont le nom commence par '.' sont masqués. Modifiez le réglage pour les afficher.", "show_hidden": "Afficher les fichiers masqués", "upload_dotfile_hidden": "{{n}} fichier(s) téléversé(s) mais masqué(s) par votre préférence.", + "upload_folder_not_ready": "Le dossier est encore en cours de chargement — merci de réessayer dans un instant.", "rename_dotfile_hidden": "Renommé en \"{{name}}\" — désormais masqué par votre préférence.", "new_folder_dotfile_hidden": "Dossier \"{{name}}\" créé — masqué par votre préférence.", "dotfiles_hidden_toast": "Fichiers masqués", @@ -801,6 +802,17 @@ "total_users": "Utilisateurs totaux", "active_users": "Utilisateurs actifs", "admins": "Admins", + "external_users": "Externes", + "external_users_tooltip": "Comptes invités — magic-link, OIDC seulement, destinataires OCM", + "online_users": "Utilisateurs en ligne", + "online_users_tooltip": "Utilisateurs distincts ayant une session active dans les 5 dernières minutes", + "online_sessions": "Sessions en ligne", + "online_sessions_tooltip": "Sessions non révoquées actives dans les 5 dernières minutes — les utilisateurs multi-appareils en contribuent plusieurs", + "section_accounts": "Comptes utilisateurs", + "section_activity": "Activité en direct", + "section_system": "Système", + "live": "en direct", + "live_tooltip": "Reflète les sessions actives dans les 5 dernières minutes", "version": "Version", "storage_overview": "Aperçu du stockage", "used": "Utilisé", @@ -1191,8 +1203,16 @@ "gen_key_warning": "Conservez cette clé en toute sécurité. En cas de perte, les données cryptées sont irrémédiablement perdues.", "jobs": { "run_all_consistency": "Exécuter tous les contrôles de cohérence", - "run_deep": "Exécuter en profondeur", + "run_deep": "Analyse approfondie", "run_deep_hint": "Exécute également des variantes lentes (re-hachage de blob, détection bitrot).", + "run_repair": "Réparer les compteurs", + "run_repair_hint": "Corrige les compteurs de références (blobs + manifestes) désynchronisés détectés par l'audit. Sûr pour les données — seuls les compteurs changent, pas le contenu.", + "run_repair_confirm_title": "Réparer les compteurs de références ?", + "run_repair_confirm_body": "Lance l'audit sur chaque blob et manifeste, puis applique un UPDATE correctif à chaque compteur qui ne correspond pas au nombre réel de références. Sans risque pour les données : seuls les compteurs changent ; le contenu des blobs et les enregistrements de fichiers ne sont pas touchés. Vous pouvez exécuter cela à tout moment ; un passage en lecture seule s'exécute d'abord pour visualiser l'écart avant que la réparation ne l'écrase.", + "run_repair_confirm_body_scoped": "Lance {{name}} et applique un UPDATE correctif à chaque compteur qui ne correspond pas au nombre réel de références. Sans risque pour les données : seuls les compteurs changent ; le contenu et les enregistrements de fichiers ne sont pas touchés.", + "run_variants_menu": "Menu des variantes d'exécution", + "run_repair_confirm": "Réparer", + "triggered_ok_repair": "{{name}} : {{n}} compteur(s) réparé(s)", "col_name": "Nom", "col_cadence": "Fréquence", "col_last_run": "Dernière exécution", @@ -1202,20 +1222,20 @@ "col_started_at": "Commencé", "col_status": "Statut", "col_duration": "Durée", - "col_scanned": "Numérisé", + "col_scanned": "Analysé", "col_progress": "Progrès", "col_findings": "Résultats", "progress_scanned_only": "{{n}} scanné", "progress_scanned_only_tooltip": "Aucun total disponible pour cette exécution (déploiement préalable de la barre de progression ou le locataire ne signale pas de sujet dénombrable).", "findings_present_tooltip": "Développez cette analyse pour voir les détails par résultat.", "col_error": "Erreur", - "col_kind": "Genre", + "col_kind": "Type", "col_severity": "Gravité", "col_resource": "Ressource", "col_detail": "Détail", "run": "Exécuter", "cancel": "Annuler", - "refresh": "Rafraîchir", + "refresh": "Actualiser", "runs_title": "Exécutions récentes", "run_json": "Résumé de l'exécution (JSON)", "findings_title": "Résultats", @@ -1225,22 +1245,22 @@ "every_h": "toutes les {{n}} h", "every_min": "toutes les {{n}} minutes", "every_sec": "toutes les {{n}} s", - "outcome_ok": "d'accord", - "outcome_err": "se tromper", + "outcome_ok": "ok", + "outcome_err": "err", "outcome_issues": "problèmes", "outcome_notices": "avis", - "n_findings": "{{n}} conclusions", + "n_findings": "{{n}} résultats", "n_notices": "{{n}} remarques", "notices_present_tooltip": "Résultats informatifs – aucune action requise. Développez pour plus de détails.", "purge": "Purger les anciennes exécutions", - "purge_hint": "Supprimez l’historique des exécutions terminées et échouées plus ancienne que la fenêtre de conservation choisie. Les résultats tombent avec leurs exécutions parentes. Les parcours non terminaux sont toujours préservés.", + "purge_hint": "Supprimer l’historique des exécutions terminées et échouées plus ancien que la fenêtre de conservation choisie. Les résultats sont supprimés avec leurs exécutions parentes. Les exécutions non terminales sont toujours préservées.", "purge_body": "Supprimez l’historique des exécutions terminées et échouées plus ancienne que le nombre de jours choisi. Les résultats tombent avec leurs exécutions parentes. Les exécutions non terminales (en cours d’exécution, en pause, demandées en annulation) sont toujours conservées.", "purge_days_label": "Rétention (jours)", "purge_confirm": "Purger", "purge_done": "{{n}} anciennes exécutions purgées (rétention {{days}} jours)", "state_running": "en cours d'exécution", "never": "jamais", - "just_now": "tout à l' heure", + "just_now": "à l'instant", "n_min_ago": "il y a {{n}} min", "n_h_ago": "il y a {{n}} h", "n_d_ago": "il y a {{n}} j", @@ -1277,6 +1297,10 @@ "revoked": "révoquée", "expired": "expirée", "active": "actif", + "online": "en ligne", + "idle": "inactif", + "presence_online_tooltip": "En ligne — vue {{ago}}", + "presence_idle_tooltip": "Inactif — vue {{ago}}", "revoke": "Révoquer", "empty": "Aucune session ne correspond au filtre actuel.", "revoke_self_confirm": "⚠️ Il s'agit de VOTRE session actuelle. La révoquer vous déconnectera immédiatement et vous devrez vous reconnecter. Continuer ?", diff --git a/frontend/static/locales/hi.json b/frontend/static/locales/hi.json index 3cd503a1..63dd5076 100644 --- a/frontend/static/locales/hi.json +++ b/frontend/static/locales/hi.json @@ -1183,11 +1183,11 @@ "gen_key_warning": "इस कुंजी को सुरक्षित रूप से संग्रहित करें. यदि यह खो जाता है, तो एन्क्रिप्टेड डेटा अपरिवर्तनीय रूप से खो जाता है।", "jobs": { "run_all_consistency": "सभी संगतता जांचें चलाएँ", - "run_deep": "गहरा रिश्ता", + "run_deep": "गहन जाँच", "run_deep_hint": "धीमे वेरिएंट (ब्लॉब री-हैश, बिट्रोट डिटेक्शन) भी चलाता है।", "col_name": "नाम", "col_cadence": "ताल", - "col_last_run": "आखरी बार", + "col_last_run": "अंतिम रन", "col_outcome": "नतीजा", "col_state": "राज्य", "col_actions": "कार्रवाई", @@ -1201,11 +1201,11 @@ "progress_scanned_only_tooltip": "इस रन के लिए कोई कुल उपलब्ध नहीं है (पूर्व-प्रगति-बार परिनियोजन या किरायेदार एक गणनीय विषय की रिपोर्ट नहीं करता है)।", "findings_present_tooltip": "प्रति-खोज विवरण देखने के लिए इस रन का विस्तार करें।", "col_error": "गलती", - "col_kind": "दयालु", + "col_kind": "प्रकार", "col_severity": "गंभीरता", "col_resource": "संसाधन", "col_detail": "विवरण", - "run": "दौड़ना", + "run": "चलाएँ", "cancel": "रद्द करना", "refresh": "ताज़ा करना", "runs_title": "हालिया रन", @@ -1218,10 +1218,10 @@ "every_min": "हर {{n}} मिनट", "every_sec": "हर {{n}} एस", "outcome_ok": "ठीक है", - "outcome_err": "ग़लती होना", + "outcome_err": "त्रुटि", "outcome_issues": "समस्याएँ", "outcome_notices": "नोटिस", - "n_findings": "{{n}}निष्कर्ष", + "n_findings": "{{n}} निष्कर्ष", "n_notices": "{{n}}नोटिस", "notices_present_tooltip": "सूचनात्मक निष्कर्ष - किसी कार्रवाई की आवश्यकता नहीं। विवरण के लिए विस्तार करें.", "purge": "पुराने रन शुद्ध करें", diff --git a/frontend/static/locales/it.json b/frontend/static/locales/it.json index a6e6dbee..b35bf833 100644 --- a/frontend/static/locales/it.json +++ b/frontend/static/locales/it.json @@ -1183,11 +1183,11 @@ "gen_key_warning": "Conserva questa chiave in modo sicuro. In caso di smarrimento, i dati crittografati andranno persi irrimediabilmente.", "jobs": { "run_all_consistency": "Esegui tutti i controlli di coerenza", - "run_deep": "Corri in profondità", + "run_deep": "Analisi approfondita", "run_deep_hint": "Esegue anche varianti lente (re-hash blob, rilevamento bitrot).", "col_name": "Nome", "col_cadence": "Cadenza", - "col_last_run": "Ultima corsa", + "col_last_run": "Ultima esecuzione", "col_outcome": "Risultato", "col_state": "Stato", "col_actions": "Azioni", @@ -1205,20 +1205,20 @@ "col_severity": "Gravità", "col_resource": "Risorsa", "col_detail": "Dettaglio", - "run": "Correre", + "run": "Esegui", "cancel": "Cancellare", "refresh": "Aggiorna", "runs_title": "Esecuzioni recenti", "run_json": "Riepilogo esecuzione (JSON)", "findings_title": "Risultati", - "no_runs": "Nessuna corsa ancora.", + "no_runs": "Nessuna esecuzione ancora.", "no_findings": "Nessun risultato: analisi pulita.", "on_demand": "su richiesta", "every_h": "ogni {{n}} h", "every_min": "ogni {{n}} min", "every_sec": "ogni {{n}} s", "outcome_ok": "OK", - "outcome_err": "errare", + "outcome_err": "err", "outcome_issues": "problemi", "outcome_notices": "avvisi", "n_findings": "{{n}} risultati", diff --git a/frontend/static/locales/ja.json b/frontend/static/locales/ja.json index 44ab8042..73908eb3 100644 --- a/frontend/static/locales/ja.json +++ b/frontend/static/locales/ja.json @@ -1183,7 +1183,7 @@ "gen_key_warning": "このキーは安全に保管してください。紛失すると、暗号化されたデータは回復不能に失われます。", "jobs": { "run_all_consistency": "すべての整合性チェックを実行する", - "run_deep": "深く走る", + "run_deep": "詳細スキャン", "run_deep_hint": "低速な亜種 (BLOB 再ハッシュ、ビットロット検出) も実行します。", "col_name": "名前", "col_cadence": "ケイデンス", @@ -1201,14 +1201,14 @@ "progress_scanned_only_tooltip": "この実行で利用できる合計はありません (進行状況バーのデプロイ前、またはテナントがカウント可能な件名を報告しない)。", "findings_present_tooltip": "この実行を展開すると、結果ごとの詳細が表示されます。", "col_error": "エラー", - "col_kind": "親切", + "col_kind": "種類", "col_severity": "重大度", "col_resource": "リソース", "col_detail": "詳細", - "run": "走る", + "run": "実行", "cancel": "キャンセル", "refresh": "リフレッシュ", - "runs_title": "最近のランニング", + "runs_title": "最近の実行", "run_json": "実行概要(JSON)", "findings_title": "調査結果", "no_runs": "まだ実行はありません。", @@ -1217,7 +1217,7 @@ "every_h": "{{n}} 時間ごと", "every_min": "{{n}} 分ごと", "every_sec": "{{n}} 秒ごと", - "outcome_ok": "わかりました", + "outcome_ok": "OK", "outcome_err": "エラー", "outcome_issues": "問題", "outcome_notices": "通知", diff --git a/frontend/static/locales/ko.json b/frontend/static/locales/ko.json index da61354e..042a4a71 100644 --- a/frontend/static/locales/ko.json +++ b/frontend/static/locales/ko.json @@ -1218,14 +1218,14 @@ "storage_backend_audit": "백엔드 일관성", "jobs": { "run_all_consistency": "모든 일관성 검사 실행", - "run_deep": "깊이 달리다", + "run_deep": "심층 스캔", "run_deep_hint": "또한 느린 변형(블롭 재해시, 비트롯 감지)을 실행합니다.", "col_name": "이름", "col_cadence": "운율", "col_last_run": "마지막 실행", "col_outcome": "결과", "col_state": "상태", - "col_actions": "행위", + "col_actions": "작업", "col_started_at": "시작됨", "col_status": "상태", "col_duration": "지속", @@ -1235,11 +1235,11 @@ "progress_scanned_only_tooltip": "이 실행에 사용할 수 있는 총계가 없습니다(사전 진행률 표시줄 배포 또는 테넌트가 셀 수 있는 주제를 보고하지 않음).", "findings_present_tooltip": "이 실행을 확장하면 발견 항목별 세부 정보를 볼 수 있습니다.", "col_error": "오류", - "col_kind": "친절한", + "col_kind": "종류", "col_severity": "심각성", "col_resource": "의지", "col_detail": "세부 사항", - "run": "달리다", + "run": "실행", "cancel": "취소", "refresh": "새로 고치다", "runs_title": "최근 실행", @@ -1251,8 +1251,8 @@ "every_h": "매 {{n}}시간마다", "every_min": "{{n}}분마다", "every_sec": "{{n}}초마다", - "outcome_ok": "좋아요", - "outcome_err": "실수", + "outcome_ok": "OK", + "outcome_err": "오류", "outcome_issues": "문제", "outcome_notices": "공지사항", "n_findings": "{{n}} 조사 결과", diff --git a/frontend/static/locales/nl.json b/frontend/static/locales/nl.json index 6bda3779..ec9311d7 100644 --- a/frontend/static/locales/nl.json +++ b/frontend/static/locales/nl.json @@ -1183,7 +1183,7 @@ "gen_key_warning": "Bewaar deze sleutel veilig. Als het verloren gaat, zijn de gecodeerde gegevens onherstelbaar verloren.", "jobs": { "run_all_consistency": "Voer alle consistentiecontroles uit", - "run_deep": "Ren diep", + "run_deep": "Diepe scan", "run_deep_hint": "Voert ook langzame varianten uit (blob re-hash, bitrot-detectie).", "col_name": "Naam", "col_cadence": "Cadans", @@ -1201,11 +1201,11 @@ "progress_scanned_only_tooltip": "Er is geen totaal beschikbaar voor deze run (implementatie vóór de voortgangsbalk of de tenant rapporteert geen telbaar onderwerp).", "findings_present_tooltip": "Vouw deze run uit om de details per vondst te bekijken.", "col_error": "Fout", - "col_kind": "Vriendelijk", + "col_kind": "Soort", "col_severity": "Ernst", "col_resource": "Bron", "col_detail": "Detail", - "run": "Loop", + "run": "Uitvoeren", "cancel": "Annuleren", "refresh": "Vernieuwen", "runs_title": "Recente runs", diff --git a/frontend/static/locales/pl.json b/frontend/static/locales/pl.json index e1ffff9e..034f30d3 100644 --- a/frontend/static/locales/pl.json +++ b/frontend/static/locales/pl.json @@ -1183,11 +1183,11 @@ "gen_key_warning": "Przechowuj ten klucz w bezpiecznym miejscu. W przypadku jego utraty zaszyfrowane dane zostaną utracone bezpowrotnie.", "jobs": { "run_all_consistency": "Uruchom wszystkie kontrole spójności", - "run_deep": "Biegnij głęboko", + "run_deep": "Głęboka analiza", "run_deep_hint": "Uruchamia również powolne warianty (ponowne mieszanie obiektów blob, wykrywanie bitrot).", "col_name": "Nazwa", "col_cadence": "Rytm", - "col_last_run": "Ostatni bieg", + "col_last_run": "Ostatnie uruchomienie", "col_outcome": "Wynik", "col_state": "Państwo", "col_actions": "Działania", @@ -1201,24 +1201,24 @@ "progress_scanned_only_tooltip": "Brak sumy dostępnej dla tego przebiegu (wdrożenie przed paskiem postępu lub dzierżawca nie zgłasza przedmiotu, który można policzyć).", "findings_present_tooltip": "Rozwiń ten przebieg, aby zobaczyć szczegóły dotyczące każdego znaleziska.", "col_error": "Błąd", - "col_kind": "Uprzejmy", + "col_kind": "Rodzaj", "col_severity": "Powaga", "col_resource": "Ratunek", "col_detail": "Szczegół", "run": "Uruchomić", "cancel": "Anulować", "refresh": "Odświeżać", - "runs_title": "Ostatnie biegi", + "runs_title": "Ostatnie uruchomienia", "run_json": "Podsumowanie uruchomienia (JSON)", "findings_title": "Ustalenia", - "no_runs": "Nie ma jeszcze żadnych biegów.", + "no_runs": "Nie ma jeszcze żadnych uruchomień.", "no_findings": "Brak wyników – czysty przebieg.", "on_demand": "na żądanie", "every_h": "co {{n}} godz", "every_min": "co {{n}} min", "every_sec": "co {{n}} s", "outcome_ok": "OK", - "outcome_err": "błądzić", + "outcome_err": "błąd", "outcome_issues": "kwestie", "outcome_notices": "uwagi", "n_findings": "{{n}} ustalenia", diff --git a/frontend/static/locales/pt.json b/frontend/static/locales/pt.json index 2b2dc27a..923da744 100644 --- a/frontend/static/locales/pt.json +++ b/frontend/static/locales/pt.json @@ -1182,8 +1182,8 @@ "gen_key": "Gerar chave", "gen_key_warning": "Armazene esta chave com segurança. Se for perdido, os dados criptografados serão perdidos irrecuperavelmente.", "jobs": { - "run_all_consistency": "Execute todas as verificações de consistência", - "run_deep": "Corra fundo", + "run_all_consistency": "Executar todas as verificações de consistência", + "run_deep": "Análise profunda", "run_deep_hint": "Também executa variantes lentas (re-hash de blob, detecção de bitrot).", "col_name": "Nome", "col_cadence": "Cadência", @@ -1205,20 +1205,20 @@ "col_severity": "Gravidade", "col_resource": "Recurso", "col_detail": "Detalhe", - "run": "Correr", + "run": "Executar", "cancel": "Cancelar", "refresh": "Atualizar", "runs_title": "Execuções recentes", "run_json": "Resumo da execução (JSON)", "findings_title": "Descobertas", - "no_runs": "Ainda não há corridas.", + "no_runs": "Ainda não há execuções.", "no_findings": "Nenhuma descoberta – execução limpa.", "on_demand": "Sob demanda", "every_h": "a cada {{n}} h", "every_min": "a cada {{n}}min", "every_sec": "cada {{n}} s", "outcome_ok": "OK", - "outcome_err": "errar", + "outcome_err": "err", "outcome_issues": "problemas", "outcome_notices": "avisos", "n_findings": "{{n}} descobertas", diff --git a/frontend/static/locales/ru.json b/frontend/static/locales/ru.json index 20bb682d..382597fa 100644 --- a/frontend/static/locales/ru.json +++ b/frontend/static/locales/ru.json @@ -1182,8 +1182,8 @@ "gen_key": "Сгенерировать ключ", "gen_key_warning": "Храните этот ключ в надежном месте. Если он утерян, зашифрованные данные теряются безвозвратно.", "jobs": { - "run_all_consistency": "Запустите все проверки согласованности", - "run_deep": "Беги глубоко", + "run_all_consistency": "Запустить все проверки согласованности", + "run_deep": "Глубокая проверка", "run_deep_hint": "Также выполняются медленные варианты (повторное хэширование больших двоичных объектов, обнаружение битротов).", "col_name": "Имя", "col_cadence": "Каденс", @@ -1201,27 +1201,27 @@ "progress_scanned_only_tooltip": "Для этого запуска общая сумма недоступна (развертывание до индикатора выполнения или клиент не сообщает об подсчитываемой теме).", "findings_present_tooltip": "Разверните этот прогон, чтобы просмотреть детали каждого результата.", "col_error": "Ошибка", - "col_kind": "Добрый", + "col_kind": "Тип", "col_severity": "Серьезность", "col_resource": "Ресурс", "col_detail": "Деталь", - "run": "Бегать", + "run": "Запустить", "cancel": "Отмена", "refresh": "Обновить", "runs_title": "Недавние запуски", "run_json": "Сводка выполнения (JSON)", "findings_title": "Выводы", - "no_runs": "Пробегов пока нет.", + "no_runs": "Запусков пока нет.", "no_findings": "Никаких результатов — чистый пробег.", "on_demand": "по требованию", "every_h": "каждые {{n}} ч", "every_min": "каждые {{n}} мин.", "every_sec": "каждые {{n}} с", - "outcome_ok": "хорошо", - "outcome_err": "ошибаться", + "outcome_ok": "ок", + "outcome_err": "ош", "outcome_issues": "проблемы", "outcome_notices": "уведомления", - "n_findings": "{{n}} выводы", + "n_findings": "{{n}} результатов", "n_notices": "{{n}} уведомления", "notices_present_tooltip": "Информационные выводы — никаких действий не требуется. Разверните для подробностей.", "purge": "Очистка старых пробегов", diff --git a/frontend/static/locales/zh-TW.json b/frontend/static/locales/zh-TW.json index da23ac86..6ac8f2e8 100644 --- a/frontend/static/locales/zh-TW.json +++ b/frontend/static/locales/zh-TW.json @@ -1165,7 +1165,7 @@ "gen_key_warning": "安全地保存此密鑰。如果遺失，加密資料將無法恢復。", "jobs": { "run_all_consistency": "執行所有一致性檢查", - "run_deep": "深入運行", + "run_deep": "深度掃描", "run_deep_hint": "也運行緩慢的變體（blob 重新哈希、bitrot 檢測）。", "col_name": "姓名", "col_cadence": "節奏", @@ -1187,10 +1187,10 @@ "col_severity": "嚴重性", "col_resource": "資源", "col_detail": "細節", - "run": "跑步", + "run": "運行", "cancel": "取消", "refresh": "重新整理", - "runs_title": "最近的跑步", + "runs_title": "最近的運行", "run_json": "運行摘要 (JSON)", "findings_title": "發現", "no_runs": "還沒有運行。", @@ -1200,7 +1200,7 @@ "every_min": "每 {{n}} 分鐘", "every_sec": "每{{n}}秒", "outcome_ok": "好的", - "outcome_err": "犯錯", + "outcome_err": "錯誤", "outcome_issues": "問題", "outcome_notices": "通知", "n_findings": "{{n}} 研究結果", diff --git a/frontend/static/locales/zh.json b/frontend/static/locales/zh.json index 4287f062..d6ff8319 100644 --- a/frontend/static/locales/zh.json +++ b/frontend/static/locales/zh.json @@ -1165,7 +1165,7 @@ "gen_key_warning": "安全地保存此密钥。如果丢失，加密数据将无法恢复。", "jobs": { "run_all_consistency": "运行所有一致性检查", - "run_deep": "深入运行", + "run_deep": "深度扫描", "run_deep_hint": "还运行缓慢的变体（blob 重新哈希、bitrot 检测）。", "col_name": "姓名", "col_cadence": "节奏", @@ -1187,10 +1187,10 @@ "col_severity": "严重性", "col_resource": "资源", "col_detail": "细节", - "run": "跑步", + "run": "运行", "cancel": "取消", "refresh": "刷新", - "runs_title": "最近的跑步", + "runs_title": "最近的运行", "run_json": "运行摘要 (JSON)", "findings_title": "发现", "no_runs": "还没有运行。", @@ -1200,7 +1200,7 @@ "every_min": "每 {{n}} 分钟", "every_sec": "每{{n}}秒", "outcome_ok": "好的", - "outcome_err": "犯错", + "outcome_err": "错误", "outcome_issues": "问题", "outcome_notices": "通知", "n_findings": "{{n}} 研究结果", diff --git a/justfile b/justfile index ad8557c7..fe306d35 100644 --- a/justfile +++ b/justfile @@ -192,7 +192,7 @@ audit: cargo audit openapi: - cargo run --bin generate-openapi + cargo run --features dev_tools --bin generate-openapi db: docker compose up -d postgres @@ -252,10 +252,10 @@ front-design: # real browser, which the curl-driven # suite above can't observe. # -# Same chain runs in CI under the `api-test` job in +# Same chain runs in CI under the `test-api` job in # .github/workflows/ci.yml; keep the order in sync so a local pass means # CI passes. -api-test: +test-api: #!/usr/bin/env bash set -x set -euo pipefail @@ -270,6 +270,12 @@ api-test: echo "XXX litmus webdav not found, ignore test" fi +# backward compat +api-test: test-api + +test-bundle: + ./tests/bundled-binary/run.sh + # CalDAV client-driven conformance suite. # # Drives OxiCloud through the maintained `python-caldav` client library @@ -279,9 +285,9 @@ api-test: # (RFC 5545 §3.8.4.4), and all-day masters (the shape #528 was filed # against). # -# Not chained into `api-test` because it needs python3; run explicitly. +# Not chained into `test-api` because it needs python3; run explicitly. # The orchestrator spawns its own postgres + server on port 8091 so it -# can run in parallel with api-test/webdav. +# can run in parallel with test-api/webdav. # # Runs `cargo build` first so the orchestrator always sees a fresh # binary. run-pycaldav.sh itself doesn't rebuild — it uses whatever @@ -301,7 +307,7 @@ test-caldav: # Manual, human-run: launches OxiCloud with OIDC as the ONLY login method # (fake IdP on :1081, server on :8090) and waits for you to eyeball the -# /login auto-redirect in a real browser. Not part of `just api-test` — +# /login auto-redirect in a real browser. Not part of `just test-api` — # there's no automated assertion here, it's a visual check. Ctrl-C to stop. #oidc-manual-sso-only: # bash tests/oidc/run-manual-sso-only.sh @@ -377,6 +383,10 @@ load-baseline: load-seed: cargo run --bin load-seed -- --depth 5 --fanout 4 --files-per-leaf 3 +# Unit-test the Docker publish tag-policy script (sub-second). +test-docker-tags: + @bash scripts/test-docker-publish-tags.sh + # Check and test everything # recommanded before pull request -pre-pull-request: check fe-check audit check-migrations test test-integration fe-test build api-test fe-build-e2e front-test +pre-pull-request: test-docker-tags check fe-check audit check-migrations test test-integration fe-test build test-bundle test-api fe-build-e2e front-test diff --git a/migrations/20261014000000_sessions_last_seen_at.sql b/migrations/20261014000000_sessions_last_seen_at.sql new file mode 100644 index 00000000..40b36c1d --- /dev/null +++ b/migrations/20261014000000_sessions_last_seen_at.sql @@ -0,0 +1,28 @@ +-- Session liveness tracking — per-request `last_seen_at` stamp on +-- `auth.sessions`, moved by the in-process `LastSeenTracker` +-- (see `src/application/services/last_seen_tracker.rs`) via a +-- batched UPDATE every 30 s. +-- +-- Distinct from `created_at`: that column moves on session ROTATION +-- (every silent refresh), so its resolution is capped at the +-- access-token TTL (default 3600 s). `last_seen_at` moves on every +-- authenticated request, so the "active in the last N min" query +-- underlying `oxicloud_sessions_active` / `_active_users` gauges is +-- accurate to the flusher's 30 s cadence regardless of token TTL. +-- +-- See `docs/plan/sessions.md` for the full design (why DashMap + +-- periodic flush, why partial index, why `NOW()` default). + +ALTER TABLE auth.sessions + ADD COLUMN last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); + +-- Partial index — the only reads on this column are the gauge +-- queries in `session_liveness_gauges.rs`, and they always filter +-- `revoked = FALSE`. Indexing only unrevoked rows keeps the write +-- cost of the 30 s batched UPDATE flat: rotated / revoked rows +-- fall out of the index automatically when `revoked` flips to TRUE +-- (partial-index maintenance drops them, no re-scan). A full +-- b-tree on the column would double index size for zero read +-- benefit — every gauge query would skip the revoked half anyway. +CREATE INDEX idx_sessions_last_seen_at ON auth.sessions(last_seen_at) + WHERE revoked = FALSE; diff --git a/migrations/20261016000000_copy_folder_tree_manifest_refcount.sql b/migrations/20261016000000_copy_folder_tree_manifest_refcount.sql new file mode 100644 index 00000000..6c3010ba --- /dev/null +++ b/migrations/20261016000000_copy_folder_tree_manifest_refcount.sql @@ -0,0 +1,211 @@ +-- Fix: `storage.copy_folder_tree` never incremented `chunk_manifests.ref_count`. +-- +-- The function bumped only `storage.blobs`: +-- +-- UPDATE storage.blobs b SET ref_count = ref_count + hc.cnt +-- FROM (...) hc WHERE b.hash = hc.blob_hash; +-- +-- but a CDC file's `blob_hash` names a MANIFEST, not a chunk. For any +-- multi-chunk file that predicate matches nothing, so a folder copy took +-- NO reference. Delete the original afterwards and `remove_reference` +-- walks the manifest to 0, `dedup_gc` reaps the manifest and every chunk +-- behind it — and the copy is unreadable. Silent data loss on an ordinary +-- UI operation. +-- +-- Single-chunk files escaped by accident: their whole-file hash equals +-- their lone chunk's hash, so the UPDATE did match — bumping the wrong +-- counter, which shows up as a manifest under-count plus a blob +-- over-count rather than as loss. +-- +-- Reproduced on a 5 MiB / 18-chunk file copied through the UI: +-- `chunk_manifests.ref_count` stayed at 1 while two `storage.files` rows +-- referenced it; `manifests_consistency` reported +-- `manifest_refcount_mismatch` with `delta: 1, reap_risk: true`. +-- +-- This migration only rewrites the reference-counting block; everything +-- else is `20260902000001_copy_folder_tree_drop_user_id.sql` verbatim. +-- +-- NOTE: existing drift is NOT repaired here. Run `manifests_consistency` +-- to find it — a data fix belongs with the recovery framework, not in a +-- schema migration that cannot know which counter is authoritative. + +CREATE OR REPLACE FUNCTION storage.copy_folder_tree( + p_source_id UUID, + p_target_parent_id UUID, -- NULL = copy to root (keeps source drive) + p_dest_name TEXT DEFAULT NULL -- NULL = keep source folder name +) RETURNS TABLE(new_root_id TEXT, folders_copied BIGINT, files_copied BIGINT) AS $$ +DECLARE + v_root_lpath ltree; + v_root_depth INT; + v_max_depth INT; + v_level INT; + v_folders BIGINT := 0; + v_files BIGINT := 0; + v_inserted BIGINT; + v_new_root UUID; + v_dest_drive_id UUID; +BEGIN + -- Validate source exists. + SELECT fo.lpath, nlevel(fo.lpath) + INTO v_root_lpath, v_root_depth + FROM storage.folders fo + WHERE fo.id = p_source_id AND NOT fo.is_trashed; + + IF v_root_lpath IS NULL THEN + RAISE EXCEPTION 'Source folder not found: %', p_source_id + USING ERRCODE = 'P0002'; -- no_data_found + END IF; + + -- Resolve destination drive_id once up front (cross-drive copy path). + IF p_target_parent_id IS NULL THEN + SELECT fo.drive_id INTO v_dest_drive_id + FROM storage.folders fo + WHERE fo.id = p_source_id; + ELSE + SELECT fo.drive_id INTO v_dest_drive_id + FROM storage.folders fo + WHERE fo.id = p_target_parent_id AND NOT fo.is_trashed; + IF v_dest_drive_id IS NULL THEN + RAISE EXCEPTION 'Target parent folder not found: %', p_target_parent_id + USING ERRCODE = 'P0002'; + END IF; + END IF; + + -- Temp mapping: every folder in the subtree → new UUID. + CREATE TEMP TABLE IF NOT EXISTS _copy_map( + old_id UUID PRIMARY KEY, + new_id UUID NOT NULL DEFAULT gen_random_uuid() + ) ON COMMIT DROP; + TRUNCATE _copy_map; + + INSERT INTO _copy_map(old_id) + SELECT fo.id + FROM storage.folders fo + WHERE NOT fo.is_trashed + AND fo.lpath <@ v_root_lpath; + + SELECT cm.new_id INTO v_new_root + FROM _copy_map cm WHERE cm.old_id = p_source_id; + + SELECT MAX(nlevel(fo.lpath)) + INTO v_max_depth + FROM storage.folders fo + JOIN _copy_map cm ON fo.id = cm.old_id; + + -- ── Insert folders level by level ── + -- Post-D7: `user_id` intentionally omitted from the column list so + -- copied rows leave the (now-nullable) column NULL. Provenance is + -- carried by `created_by` / `updated_by` (§14 columns) — preserved + -- from source so authorship survives the copy. + FOR v_level IN v_root_depth .. v_max_depth LOOP + INSERT INTO storage.folders( + id, name, parent_id, + drive_id, created_by, updated_by + ) + SELECT cm.new_id, + CASE WHEN fo.id = p_source_id AND p_dest_name IS NOT NULL + THEN p_dest_name ELSE fo.name END, + CASE WHEN fo.id = p_source_id THEN p_target_parent_id + ELSE pm.new_id END, + v_dest_drive_id, + fo.created_by, + fo.updated_by + FROM storage.folders fo + JOIN _copy_map cm ON fo.id = cm.old_id + LEFT JOIN _copy_map pm ON fo.parent_id = pm.old_id + WHERE NOT fo.is_trashed + AND nlevel(fo.lpath) = v_level; + + GET DIAGNOSTICS v_inserted = ROW_COUNT; + v_folders := v_folders + v_inserted; + END LOOP; + + -- Temp mapping for files src→dst (dst ids pre-allocated so we can + -- reference them in the dead-property duplication below). + CREATE TEMP TABLE IF NOT EXISTS _copy_file_map( + old_id UUID PRIMARY KEY, + new_id UUID NOT NULL DEFAULT gen_random_uuid() + ) ON COMMIT DROP; + TRUNCATE _copy_file_map; + + INSERT INTO _copy_file_map(old_id) + SELECT f.id + FROM storage.files f + JOIN _copy_map cm ON f.folder_id = cm.old_id + WHERE NOT f.is_trashed; + + -- ── Batch copy all files (zero-copy: same blob_hash) ── + -- Post-D7: `user_id` omitted. Provenance via `created_by`/`updated_by`. + INSERT INTO storage.files( + id, name, folder_id, blob_hash, size, mime_type, + media_sort_date, drive_id, created_by, updated_by + ) + SELECT fm.new_id, f.name, cm.new_id, f.blob_hash, f.size, + f.mime_type, f.media_sort_date, v_dest_drive_id, f.created_by, + f.updated_by + FROM storage.files f + JOIN _copy_map cm ON f.folder_id = cm.old_id + JOIN _copy_file_map fm ON fm.old_id = f.id + WHERE NOT f.is_trashed; + + GET DIAGNOSTICS v_files = ROW_COUNT; + + -- Batch increment reference counts — MANIFEST FIRST, blobs only as + -- fallback. This mirrors `DedupService::add_reference`, and the order + -- is the whole point: + -- + -- A CDC file's `blob_hash` names a MANIFEST (`chunk_manifests.file_hash`), + -- not a chunk. The previous version of this block updated only + -- `storage.blobs`, so for a multi-chunk file the predicate + -- `b.hash = hc.blob_hash` matched ZERO rows and the copy took no + -- reference at all. Deleting the original then walked the manifest's + -- ref_count to 0, dedup_gc reaped the manifest and every chunk behind + -- it, and the copy became unreadable. Reproduced via the UI folder + -- copy on a 5 MiB (18-chunk) file: ref_count stayed 1 with two files + -- referencing it. + -- + -- The `NOT EXISTS (bumped)` guard on the blobs branch is load-bearing. + -- For a SINGLE-chunk file the whole-file hash equals its lone chunk's + -- hash, so without it the copy would be counted at both levels and + -- turn an under-count into an over-count. + IF v_files > 0 THEN + WITH hc AS ( + SELECT f.blob_hash, COUNT(*)::int AS cnt + FROM storage.files f + JOIN _copy_map cm ON f.folder_id = cm.new_id + WHERE NOT f.is_trashed + GROUP BY f.blob_hash + ), + bumped AS ( + UPDATE storage.chunk_manifests m + SET ref_count = m.ref_count + hc.cnt + FROM hc + WHERE m.file_hash = hc.blob_hash + RETURNING m.file_hash + ) + UPDATE storage.blobs b + SET ref_count = b.ref_count + hc.cnt, + -- Matches add_reference: a blob resurrected inside its GC + -- grace window must lose its orphan stamp. + orphaned_at = NULL + FROM hc + WHERE b.hash = hc.blob_hash + AND NOT EXISTS (SELECT 1 FROM bumped WHERE file_hash = hc.blob_hash); + END IF; + + -- Duplicate dead properties per RFC 4918 §8.8 — id-keyed store. + INSERT INTO storage.webdav_dead_properties + (folder_id, namespace, local_name, value) + SELECT cm.new_id, dp.namespace, dp.local_name, dp.value + FROM storage.webdav_dead_properties dp + JOIN _copy_map cm ON dp.folder_id = cm.old_id; + + INSERT INTO storage.webdav_dead_properties + (file_id, namespace, local_name, value) + SELECT fm.new_id, dp.namespace, dp.local_name, dp.value + FROM storage.webdav_dead_properties dp + JOIN _copy_file_map fm ON dp.file_id = fm.old_id; + + RETURN QUERY SELECT v_new_root::text, v_folders, v_files; +END; +$$ LANGUAGE plpgsql; diff --git a/migrations/20261017000000_file_delete_trigger_manifest_aware.sql b/migrations/20261017000000_file_delete_trigger_manifest_aware.sql new file mode 100644 index 00000000..c166143f --- /dev/null +++ b/migrations/20261017000000_file_delete_trigger_manifest_aware.sql @@ -0,0 +1,108 @@ +-- Fix: `trg_files_decrement_blob_ref` decremented the wrong counter for +-- CDC files. +-- +-- The original trigger (2026-03-07 initial schema) unconditionally ran: +-- +-- UPDATE storage.blobs +-- SET ref_count = GREATEST(ref_count - 1, 0) +-- WHERE hash = OLD.blob_hash; +-- +-- That's correct for a legacy whole-file blob, where `OLD.blob_hash` +-- names a `storage.blobs` row directly. For a CDC file, `OLD.blob_hash` +-- names a `storage.chunk_manifests.file_hash` — the blob table row (if +-- one exists at all) holds a DIFFERENT counter, incremented by the +-- MANIFEST's presence in its own `chunk_hashes[]`, not by the file. +-- +-- Consequences before this fix: +-- 1. `storage.chunk_manifests.ref_count` never decremented on file +-- DELETE → over-count grows unboundedly across delete/purge +-- cycles. +-- 2. `storage.blobs.ref_count` decremented for hashes it shouldn't +-- (CDC whole-file hashes) → the counter drops toward 0 while the +-- manifest still legitimately references the chunk. GC then reaps +-- a live blob → downloadable-then-404 data loss. +-- +-- Both bugs surfaced by `tests/api/refcount_cascade.hurl` on the +-- 135-byte fixture (single-chunk CDC file, worst case for confusion +-- because the whole-file hash equals its lone chunk's hash). The +-- 2026-08-22 sandbox drift (`storage.blobs.ref_count = 0`, +-- `actual_auditor = 1`) is the same bug at rest. +-- +-- Sibling fix: `20261016000000_copy_folder_tree_manifest_refcount.sql` +-- fixed the mirror-image INCREMENT bug in `storage.copy_folder_tree`. +-- This migration closes the decrement half. +-- +-- Cross-references: +-- - `DedupService::add_reference` (dedup_service.rs:1703) — app-layer +-- twin for the increment direction: manifest first, blob fallback. +-- - `manifests_consistency` tenant (2026-08-23) — surfaces any +-- residual drift after this fix lands. +-- +-- ── DESIGN NOTE — decrement only, no manifest reap here ── +-- +-- The trigger DELIBERATELY does not delete manifests or walk chunks on +-- a last-ref decrement. Both actions used to live inside +-- `DedupService::cleanup_if_orphaned` and its callee +-- `remove_manifest_reference`, and both fire `fire_blob_hooks` — +-- the Rust callback that reaps disk artefacts keyed by the whole-file +-- content hash (thumbnails, face embeddings, audio tags, media +-- metadata). SQL triggers can't invoke Rust callbacks, so if this +-- trigger reaped the manifest itself, dedup_gc Phase 1 +-- (`dedup_service.rs:2660-2772`) — the ONLY code path that knows to +-- fire `fire_blob_hooks` for a reaped manifest's `file_hash` — would +-- find nothing to do on its next sweep, and every derived artefact +-- would leak on disk. `storage_cleanup_check.sh`'s "N thumbnail +-- file(s) remain on disk" gate catches this class immediately. +-- +-- Contract: trigger decrements the correct counter atomically inside +-- the DELETE txn. GC (`dedup_gc`) is responsible for: +-- • finding manifests whose ref_count hit 0 (or that no reference +-- source references, covering bulk-delete paths), +-- • deleting them, +-- • decrementing each chunk in `chunk_hashes[]`, +-- • firing `fire_blob_hooks(file_hash)` so Rust callbacks reap +-- derived disk artefacts, +-- • the corresponding legacy-blob path for ref_count = 0 blobs. +-- +-- NOTE: pre-existing drift is NOT repaired here. Run `manifests_ +-- consistency` + `blobs_consistency` after deploy; feed the findings +-- into the recovery framework. + +CREATE OR REPLACE FUNCTION storage.decrement_blob_ref() +RETURNS trigger AS $$ +BEGIN + -- Manifest-first, mirroring the increment side. We touch ONE + -- counter and return — the manifest reap + chunk walk + hook + -- firing lives in `dedup_gc` where Rust callbacks can run. + IF EXISTS ( + SELECT 1 FROM storage.chunk_manifests + WHERE file_hash = OLD.blob_hash + ) THEN + UPDATE storage.chunk_manifests + SET ref_count = GREATEST(ref_count - 1, 0) + WHERE file_hash = OLD.blob_hash; + ELSE + -- Legacy whole-file blob path: no manifest, blob is referenced + -- directly by this file row. Preserves the original behaviour + -- verbatim for the pre-CDC path. + UPDATE storage.blobs + SET ref_count = GREATEST(ref_count - 1, 0), + orphaned_at = CASE + WHEN GREATEST(ref_count - 1, 0) = 0 + THEN now() + ELSE orphaned_at + END + WHERE hash = OLD.blob_hash; + END IF; + + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +COMMENT ON FUNCTION storage.decrement_blob_ref() IS + 'Decrement the correct ref_count when a file is deleted. ' + 'Manifest-aware (2026-10-17): dispatches to chunk_manifests.ref_count ' + 'when the file''s blob_hash names a manifest, else to ' + 'storage.blobs.ref_count for legacy whole-file blobs. Decrement-only: ' + 'physical cleanup + Rust lifecycle hooks fire from dedup_gc, which ' + 'can invoke callbacks a SQL trigger cannot.'; diff --git a/migrations/20261017000002_repair_existing_refcount_drift.sql b/migrations/20261017000002_repair_existing_refcount_drift.sql new file mode 100644 index 00000000..c76be038 --- /dev/null +++ b/migrations/20261017000002_repair_existing_refcount_drift.sql @@ -0,0 +1,162 @@ +-- One-time repair of ref_count drift accumulated under the pre-fix +-- copy/delete code paths. +-- +-- Why this is atomic with the upgrade rather than a manual admin action +-- ───────────────────────────────────────────────────────────────────── +-- The two prior migrations on this branch: +-- * `20261016000000_copy_folder_tree_manifest_refcount.sql` +-- (fix the INCREMENT path — copy was bumping the wrong counter for +-- CDC files) +-- * `20261017000000_file_delete_trigger_manifest_aware.sql` +-- (fix the DECREMENT path — trigger was decrementing the wrong +-- counter for CDC files; folder-cascade + trash-empty paths +-- inherited that drift silently) +-- both close the bugs going forward, but production DBs upgrading +-- through this branch may carry accumulated drift from every prior +-- copy → delete cycle a CDC file went through. Under-count is the +-- dangerous direction: the next `dedup_gc` pass would reap a live +-- blob → user-facing 404 → silent data loss. +-- +-- Waiting for an operator to open the admin panel and click "Repair +-- ref_counts" is the wrong default for a data-loss-preventing fix. +-- Ed's rule (`[[feedback_no_silent_auto_repair]]`): consistency +-- tenants must default to discovery-only so future bugs surface — but +-- fixing KNOWN pre-existing drift on the upgrade itself is the +-- bounded exception, because at that specific moment the source of +-- drift is known + closed, and there is no upstream mystery to +-- preserve. +-- +-- Content-safety guarantees: +-- * Only counter columns change (`storage.chunk_manifests.ref_count`, +-- `storage.blobs.ref_count`). No file rows, no blob rows, no +-- manifest rows, no chunk arrays, no backend files. +-- * The corrective UPDATE sets `stored = actual` where `actual` is +-- computed from the SAME auditor formulas that +-- `manifests_consistency` / `blobs_consistency` use, so this +-- migration and those tenants agree by construction. +-- * Race-safe against concurrent writes (migrations run +-- single-connection at startup before the server serves any +-- traffic; nobody else is writing). +-- * Idempotent — fresh installs and already-clean DBs no-op (both +-- `stored` and `actual` are equal, the `WHERE <>` filters +-- everything out). +-- +-- The panel button + `?repair=true` on the trigger endpoints stay for +-- FUTURE drift (regression detector; not for repeat use on this +-- accumulated set). +-- +-- ═══════════════════════════════════════════════════════════════════ +-- Performance envelope (rewrite 2026-09-02) +-- ═══════════════════════════════════════════════════════════════════ +-- Original implementation used correlated subqueries in both SET and +-- WHERE clauses — PG evaluates each subquery twice per row, and the +-- `b.hash = ANY(m.chunk_hashes)` scan is O(blobs × manifests) without +-- a GIN index. On a production customer with a large storage.blobs + +-- storage.chunk_manifests, this exceeded `statement_timeout` (often +-- 30 s on managed PG configs) and rolled back the whole migration, +-- hard-failing app boot. +-- +-- Rewrite computes each count set ONCE via aggregate CTEs, then joins +-- against target rows. Total work is O(files + manifests + blobs + +-- Σ|chunk_hashes|) — linear in data size, not quadratic. Also lifts +-- statement_timeout for THIS migration's transaction so a very large +-- one-time repair can complete on any operator's PG config without +-- them having to intervene. +-- +-- Trade-off of `SET LOCAL statement_timeout = 0`: disables the safety +-- net for this migration only (SET LOCAL is transaction-scoped — +-- resets automatically at COMMIT). Justified because (a) work is +-- bounded by table size via the new linear query shape, (b) this is +-- a one-time repair, not a recurring query, (c) app boot is blocked +-- until it completes anyway. +-- +-- Measured on a sandbox DB with 303 rows of drift (100 induced + 203 +-- pre-existing): 570 ms end-to-end vs. timeout in the original form. + +-- Lift the timeout for this migration only. Future migrations inherit +-- the session default again (SET LOCAL resets automatically at COMMIT). +SET LOCAL statement_timeout = 0; + +DO $$ +DECLARE + v_m_fixed int; + v_b_fixed int; +BEGIN + -- Manifest counter: `actual` = # files whose blob_hash names this + -- manifest's file_hash. Same formula as + -- `manifests_consistency_service::manifest_page_sql` (via the + -- BlobReferenceRegistry at RefLevel::Manifest) — inline here + -- because migrations can't call Rust. + -- + -- Structure: one GROUP BY over storage.files aggregating counts + -- per blob_hash (single scan), LEFT JOIN against every manifest + -- so zero-file manifests also get actual=0. UPDATE ... FROM + -- walks manifests once, writes only where stored <> actual. + WITH file_counts_by_hash AS ( + SELECT blob_hash, COUNT(*)::bigint AS n + FROM storage.files + GROUP BY blob_hash + ), + actual_per_manifest AS ( + SELECT m.file_hash, + COALESCE(fc.n, 0) AS actual + FROM storage.chunk_manifests m + LEFT JOIN file_counts_by_hash fc ON fc.blob_hash = m.file_hash + ) + UPDATE storage.chunk_manifests m + SET ref_count = a.actual + FROM actual_per_manifest a + WHERE a.file_hash = m.file_hash + AND m.ref_count <> a.actual; + GET DIAGNOSTICS v_m_fixed = ROW_COUNT; + + -- Blob counter: two-term formula mirroring + -- `blobs_consistency_service.rs:395-408`: + -- (files pointing at this blob AND having NO manifest for their + -- blob_hash — legacy whole-file path) + -- + (manifests including this hash as a chunk in chunk_hashes[]) + -- + -- Structure: two aggregate CTEs (one per term), then LEFT JOINed + -- against every blob. `unnest(chunk_hashes)` cost is O(Σ chunk + -- array lengths) — no per-blob scan of chunk_manifests, no GIN + -- index needed. + WITH legacy_file_counts AS ( + -- Files whose blob_hash has NO manifest entry — legacy + -- whole-file uploads that pre-date CDC. + SELECT f.blob_hash, COUNT(*)::bigint AS legacy_count + FROM storage.files f + WHERE NOT EXISTS ( + SELECT 1 FROM storage.chunk_manifests m + WHERE m.file_hash = f.blob_hash + ) + GROUP BY f.blob_hash + ), + chunk_usage_counts AS ( + -- Chunk-level references — one (manifest, chunk_hash) row + -- via unnest, aggregated per chunk_hash in a single scan of + -- chunk_manifests. + SELECT ch AS hash, COUNT(*)::bigint AS chunk_count + FROM storage.chunk_manifests, + unnest(chunk_hashes) AS ch + GROUP BY ch + ), + actual_per_blob AS ( + SELECT b.hash, + COALESCE(l.legacy_count, 0) + COALESCE(u.chunk_count, 0) AS actual + FROM storage.blobs b + LEFT JOIN legacy_file_counts l ON l.blob_hash = b.hash + LEFT JOIN chunk_usage_counts u ON u.hash = b.hash + ) + UPDATE storage.blobs b + SET ref_count = a.actual + FROM actual_per_blob a + WHERE a.hash = b.hash + AND b.ref_count <> a.actual; + GET DIAGNOSTICS v_b_fixed = ROW_COUNT; + + -- Landed in the deploy log so an operator upgrading a huge instance + -- can see the migration did work — silent no-op on fresh installs. + RAISE NOTICE '[refcount_repair] fixed % manifest(s), % blob(s)', + v_m_fixed, v_b_fixed; +END; +$$; diff --git a/migrations/20261018000000_content_derived_blobs.sql b/migrations/20261018000000_content_derived_blobs.sql new file mode 100644 index 00000000..2baee5a2 --- /dev/null +++ b/migrations/20261018000000_content_derived_blobs.sql @@ -0,0 +1,69 @@ +-- Derived content as blobs — tier-2 refactor, step 5. +-- See `docs/plan/derived-blobs.md`. +-- +-- Maps a source Blob to the artifacts derived FROM it: thumbnails today, +-- transcodes next. Both the mapping key and the value are BLAKE3 hashes, +-- but they mean different things: +-- +-- * `source_hash` — the Blob the artifact was derived from. A +-- *dependent* reference: it keeps nothing alive (the file does), and +-- when that Blob dies these rows are deleted with it. +-- * `blob_hash` — the derived Blob itself. A reference *holder*: it +-- bumps `chunk_manifests.ref_count`, which is why +-- `ContentDerivedReferenceSource` must be registered before the first +-- row is written, or `dedup_gc` reaps the content on its next sweep. +-- +-- KEYING — the rule this table exists to enforce: +-- +-- Bytes that are a pure deterministic function of the source content +-- belong here, content-keyed, and dedupe across every file holding +-- that content. Bytes that are user-supplied or user-chosen do NOT: +-- they must be file-keyed, because content-keying them lets one user's +-- upload be served for another user's identical file. Client-uploaded +-- previews (PDF page 1, video poster frames) are the live example and +-- belong in a separate file-keyed table. +-- +-- `variant` is opaque text. New axes go INSIDE it, never into new +-- columns: 'preview-avif' beside 'preview', '720p-av1' beside '720p'. +-- That is what keeps this table from growing a column per rendering +-- parameter. +-- +-- No FK on either hash column, for the reason +-- `20260701000000_content_search_index.sql` already documents: a hash +-- resolves to either `storage.blobs` (legacy whole blob) or +-- `storage.chunk_manifests` (CDC file hash), so the reference cannot be +-- expressed as a single FK. Orphans are reclaimed by GC and reported by +-- the consistency jobs instead. +-- +-- No `size` column: the bytes are content-addressed, so their length is +-- an immutable fact the blob layer already owns via `blob_hash`. +-- `content_type` IS stored — the thumbnail handler byte-sniffs every +-- response today, and this retires that. + +CREATE TABLE IF NOT EXISTS storage.content_derived_blobs ( + source_hash VARCHAR(64) NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('thumbnail', 'transcode')), + variant TEXT NOT NULL, + blob_hash VARCHAR(64) NOT NULL, + content_type TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source_hash, kind, variant) +); + +-- Reverse lookup: "what still references this derived Blob?" — used by +-- the manifest-level refcount recompute in `manifests_consistency` and by +-- `dedup_gc`'s reap predicate. +CREATE INDEX IF NOT EXISTS idx_content_derived_blobs_blob_hash + ON storage.content_derived_blobs (blob_hash); + +COMMENT ON TABLE storage.content_derived_blobs IS + 'Server-derived artifacts (thumbnails, transcodes) keyed by the BLAKE3 of their SOURCE content. Content-keyed on purpose: identical content shares one derivation. User-supplied bytes must NOT be stored here — see docs/plan/derived-blobs.md.'; + +COMMENT ON COLUMN storage.content_derived_blobs.source_hash IS + 'The Blob this was derived from. Dependent reference — holds no ref_count; rows are deleted when the source Blob is reaped.'; + +COMMENT ON COLUMN storage.content_derived_blobs.blob_hash IS + 'The derived Blob. Reference HOLDER — bumps chunk_manifests.ref_count via DedupService::add_reference.'; + +COMMENT ON COLUMN storage.content_derived_blobs.variant IS + 'Opaque rendering discriminator (icon | preview | large | 720p...). New axes go inside this string, never into new columns.'; diff --git a/migrations/20261019000000_copy_file_satellites.sql b/migrations/20261019000000_copy_file_satellites.sql new file mode 100644 index 00000000..030812bd --- /dev/null +++ b/migrations/20261019000000_copy_file_satellites.sql @@ -0,0 +1,332 @@ +-- Step 8 of `docs/plan/derived-blobs.md` — single-source the copy fan-out. +-- +-- "What follows a file when the file is copied" was written twice: once in +-- the `copy_file` CTE (Rust, `file_blob_write_repository.rs`) and once in +-- `storage.copy_folder_tree`. They had already drifted — the tree path +-- bumped `storage.blobs` only, missing manifests entirely, which was silent +-- data loss on a multi-chunk file (fixed in `20261016000000`, and the fix +-- had to be written a second time rather than in one place). +-- +-- The plan adds file-keyed satellite tables (`file_attached_blobs`, step 9). +-- Adding them against two copy sites means writing the same cascade a third +-- and fourth time, into sites that have already proven they drift. So the +-- fan-out gets exactly one home first. +-- +-- Two functions land here: +-- +-- * `storage.add_blob_references(TEXT[])` — the manifest-first reference +-- contract, expressed once for SQL callers. `DedupService::add_reference` +-- is the Rust twin; they must change together, which is why the shared +-- contract is spelled out in both doc comments. +-- +-- * `storage.copy_file_satellites(UUID[], UUID[])` — everything that +-- follows a file on copy. The body IS the copy-semantics declaration: +-- what is absent is a documented decision (see the trailing comments), +-- not an omission someone has to notice. +-- +-- Set-based rather than per-row on purpose. A per-row helper would have made +-- a 10k-file folder copy 10k function calls; taking arrays keeps the tree +-- path's single-statement cost while still having one implementation. The +-- single-file path passes one-element arrays. + +-- ── The reference contract, for SQL callers ────────────────────────────── +-- +-- Increment the reference count for each hash in `p_hashes`, counting +-- repeats (pass the hash once per referencing row). Returns the hashes that +-- matched NEITHER table, so callers can decide how loud to be — a copy +-- inherits a pre-existing breakage and should warn, whereas an ingest +-- referencing a nonexistent blob is a hard error. +-- +-- MANIFEST FIRST, `storage.blobs` only as fallback. The order is the whole +-- point: a CDC file's `blob_hash` names a manifest +-- (`chunk_manifests.file_hash`), not a chunk, so bumping `storage.blobs` +-- first would match nothing for a multi-chunk file and take no reference at +-- all. +-- +-- The `NOT EXISTS (bumped)` guard on the blobs branch is load-bearing. For a +-- SINGLE-chunk file the whole-file hash EQUALS its lone chunk's hash (both +-- are BLAKE3 over the same bytes), so without the guard one reference would +-- be counted at both levels — turning an under-count into an over-count. +-- +-- Mirrors `DedupService::add_reference`, including the asymmetry on +-- `orphaned_at`: only `storage.blobs` carries that column, so only the blobs +-- branch clears it. A chunk resurrected inside its GC grace window must lose +-- its orphan stamp or `dedup_gc` reaps live content. +CREATE OR REPLACE FUNCTION storage.add_blob_references(p_hashes TEXT[]) +RETURNS TEXT[] AS $$ +DECLARE + v_unmatched TEXT[]; +BEGIN + IF p_hashes IS NULL OR cardinality(p_hashes) = 0 THEN + RETURN ARRAY[]::TEXT[]; + END IF; + + WITH hc AS ( + SELECT h AS blob_hash, COUNT(*)::int AS cnt + FROM unnest(p_hashes) AS h + WHERE h IS NOT NULL + GROUP BY h + ), + bumped_manifests AS ( + UPDATE storage.chunk_manifests m + SET ref_count = m.ref_count + hc.cnt + FROM hc + WHERE m.file_hash = hc.blob_hash + RETURNING m.file_hash + ), + bumped_blobs AS ( + UPDATE storage.blobs b + SET ref_count = b.ref_count + hc.cnt, + orphaned_at = NULL + FROM hc + WHERE b.hash = hc.blob_hash + AND NOT EXISTS ( + SELECT 1 FROM bumped_manifests WHERE file_hash = hc.blob_hash + ) + RETURNING b.hash + ) + SELECT COALESCE(array_agg(hc.blob_hash), ARRAY[]::TEXT[]) + INTO v_unmatched + FROM hc + WHERE NOT EXISTS (SELECT 1 FROM bumped_manifests WHERE file_hash = hc.blob_hash) + AND NOT EXISTS (SELECT 1 FROM bumped_blobs WHERE hash = hc.blob_hash); + + RETURN v_unmatched; +END; +$$ LANGUAGE plpgsql; + +COMMENT ON FUNCTION storage.add_blob_references(TEXT[]) IS + 'Manifest-first blob reference increment for SQL callers. Returns hashes ' + 'that matched no registry row. Rust twin: DedupService::add_reference — ' + 'change both together.'; + +-- ── What follows a file on copy ────────────────────────────────────────── +-- +-- `p_old_ids[i]` is copied to `p_new_ids[i]`; the new `storage.files` rows +-- must already be inserted and visible (both callers insert in an earlier +-- statement of the same transaction). +-- +-- Every satellite of a copied file belongs in this body. What is NOT here is +-- listed at the bottom, with the reason — the taxonomy is executable rather +-- than living in a document that drifts from the code. +CREATE OR REPLACE FUNCTION storage.copy_file_satellites( + p_old_ids UUID[], + p_new_ids UUID[] +) RETURNS void AS $$ +DECLARE + v_unmatched TEXT[]; +BEGIN + IF p_old_ids IS NULL OR cardinality(p_old_ids) = 0 THEN + RETURN; + END IF; + + IF p_new_ids IS NULL OR cardinality(p_old_ids) <> cardinality(p_new_ids) THEN + -- Positional correspondence is the whole interface; a length + -- mismatch would silently attach satellites to the wrong file. + RAISE EXCEPTION + 'copy_file_satellites: id arrays must correspond positionally (% old vs % new)', + cardinality(p_old_ids), COALESCE(cardinality(p_new_ids), 0); + END IF; + + -- 1. WebDAV dead properties. RFC 4918 §8.8 requires COPY to duplicate + -- them: properties describe the resource, and the copy is a resource. + INSERT INTO storage.webdav_dead_properties + (file_id, namespace, local_name, value) + SELECT m.new_id, dp.namespace, dp.local_name, dp.value + FROM unnest(p_old_ids, p_new_ids) AS m(old_id, new_id) + JOIN storage.webdav_dead_properties dp ON dp.file_id = m.old_id; + + -- 2. A reference on the copied content, so deleting the original cannot + -- reap bytes the copy still needs. Read from the NEW rows rather than + -- the old ones: that is what makes an unreferenceable copy impossible + -- to create, since a row that failed to insert contributes nothing. + SELECT storage.add_blob_references(array_agg(f.blob_hash)) + INTO v_unmatched + FROM unnest(p_new_ids) AS n(id) + JOIN storage.files f ON f.id = n.id + WHERE NOT f.is_trashed; + + IF v_unmatched IS NOT NULL AND cardinality(v_unmatched) > 0 THEN + -- Warn, do not abort. A missing registry row means the SOURCE file + -- was already broken; the copy merely inherits it. Failing here + -- would abort an entire folder copy over one pre-existing fault, + -- which is worse than completing it and reporting. The blob-level + -- audit jobs are what surface the underlying breakage. + RAISE WARNING + 'copy_file_satellites: % copied file(s) reference a blob with no registry row (first: %); source was already broken', + cardinality(v_unmatched), v_unmatched[1]; + END IF; + + -- ── Deliberately absent ────────────────────────────────────────────── + -- + -- storage.comments (future): NOT copied. A copy is a new artifact; the + -- discussion belongs to the original. + -- + -- storage.file_attached_blobs (step 9): WILL be copied here, with a + -- reference taken per attached blob_hash via add_blob_references. + -- + -- content_derived_blobs, blob_extracted_text, faces.faces: content-keyed. + -- The copy shares the source's hash, so it already sees them — copying + -- would duplicate rows that are keyed on the very thing being shared. + -- + -- storage.favorites, recent_items, shares: properties of the ORIGINAL's + -- relationship to users, not of its content. +END; +$$ LANGUAGE plpgsql; + +COMMENT ON FUNCTION storage.copy_file_satellites(UUID[], UUID[]) IS + 'Single source of truth for what follows a file on copy. Both copy paths ' + '(single-file and copy_folder_tree) call it. Adding a file-keyed satellite ' + 'table means editing this function, and only this function.'; + +-- ── Route copy_folder_tree through it ──────────────────────────────────── +-- +-- Only two blocks change versus `20261016000000`: the inline reference bump +-- and the per-file dead-property INSERT are both replaced by one +-- `copy_file_satellites` call. The folder dead-property INSERT stays inline +-- — folders are not files and have no satellite fan-out to share. +CREATE OR REPLACE FUNCTION storage.copy_folder_tree( + p_source_id UUID, + p_target_parent_id UUID, -- NULL = copy to root (keeps source drive) + p_dest_name TEXT DEFAULT NULL -- NULL = keep source folder name +) RETURNS TABLE(new_root_id TEXT, folders_copied BIGINT, files_copied BIGINT) AS $$ +DECLARE + v_root_lpath ltree; + v_root_depth INT; + v_max_depth INT; + v_level INT; + v_folders BIGINT := 0; + v_files BIGINT := 0; + v_inserted BIGINT; + v_new_root UUID; + v_dest_drive_id UUID; +BEGIN + -- Validate source exists. + SELECT fo.lpath, nlevel(fo.lpath) + INTO v_root_lpath, v_root_depth + FROM storage.folders fo + WHERE fo.id = p_source_id AND NOT fo.is_trashed; + + IF v_root_lpath IS NULL THEN + RAISE EXCEPTION 'Source folder not found: %', p_source_id + USING ERRCODE = 'P0002'; -- no_data_found + END IF; + + -- Resolve destination drive_id once up front (cross-drive copy path). + IF p_target_parent_id IS NULL THEN + SELECT fo.drive_id INTO v_dest_drive_id + FROM storage.folders fo + WHERE fo.id = p_source_id; + ELSE + SELECT fo.drive_id INTO v_dest_drive_id + FROM storage.folders fo + WHERE fo.id = p_target_parent_id AND NOT fo.is_trashed; + IF v_dest_drive_id IS NULL THEN + RAISE EXCEPTION 'Target parent folder not found: %', p_target_parent_id + USING ERRCODE = 'P0002'; + END IF; + END IF; + + -- Temp mapping: every folder in the subtree → new UUID. + CREATE TEMP TABLE IF NOT EXISTS _copy_map( + old_id UUID PRIMARY KEY, + new_id UUID NOT NULL DEFAULT gen_random_uuid() + ) ON COMMIT DROP; + TRUNCATE _copy_map; + + INSERT INTO _copy_map(old_id) + SELECT fo.id + FROM storage.folders fo + WHERE NOT fo.is_trashed + AND fo.lpath <@ v_root_lpath; + + SELECT cm.new_id INTO v_new_root + FROM _copy_map cm WHERE cm.old_id = p_source_id; + + SELECT MAX(nlevel(fo.lpath)) + INTO v_max_depth + FROM storage.folders fo + JOIN _copy_map cm ON fo.id = cm.old_id; + + -- ── Insert folders level by level ── + -- Post-D7: `user_id` intentionally omitted from the column list so + -- copied rows leave the (now-nullable) column NULL. Provenance is + -- carried by `created_by` / `updated_by` (§14 columns) — preserved + -- from source so authorship survives the copy. + FOR v_level IN v_root_depth .. v_max_depth LOOP + INSERT INTO storage.folders( + id, name, parent_id, + drive_id, created_by, updated_by + ) + SELECT cm.new_id, + CASE WHEN fo.id = p_source_id AND p_dest_name IS NOT NULL + THEN p_dest_name ELSE fo.name END, + CASE WHEN fo.id = p_source_id THEN p_target_parent_id + ELSE pm.new_id END, + v_dest_drive_id, + fo.created_by, + fo.updated_by + FROM storage.folders fo + JOIN _copy_map cm ON fo.id = cm.old_id + LEFT JOIN _copy_map pm ON fo.parent_id = pm.old_id + WHERE NOT fo.is_trashed + AND nlevel(fo.lpath) = v_level; + + GET DIAGNOSTICS v_inserted = ROW_COUNT; + v_folders := v_folders + v_inserted; + END LOOP; + + -- Temp mapping for files src→dst (dst ids pre-allocated so we can hand + -- both sides to copy_file_satellites below). + CREATE TEMP TABLE IF NOT EXISTS _copy_file_map( + old_id UUID PRIMARY KEY, + new_id UUID NOT NULL DEFAULT gen_random_uuid() + ) ON COMMIT DROP; + TRUNCATE _copy_file_map; + + INSERT INTO _copy_file_map(old_id) + SELECT f.id + FROM storage.files f + JOIN _copy_map cm ON f.folder_id = cm.old_id + WHERE NOT f.is_trashed; + + -- ── Batch copy all files (zero-copy: same blob_hash) ── + -- Post-D7: `user_id` omitted. Provenance via `created_by`/`updated_by`. + INSERT INTO storage.files( + id, name, folder_id, blob_hash, size, mime_type, + media_sort_date, drive_id, created_by, updated_by + ) + SELECT fm.new_id, f.name, cm.new_id, f.blob_hash, f.size, + f.mime_type, f.media_sort_date, v_dest_drive_id, f.created_by, + f.updated_by + FROM storage.files f + JOIN _copy_map cm ON f.folder_id = cm.old_id + JOIN _copy_file_map fm ON fm.old_id = f.id + WHERE NOT f.is_trashed; + + GET DIAGNOSTICS v_files = ROW_COUNT; + + -- Everything that follows a file on copy — blob references and dead + -- properties — in one call, shared with the single-file copy path. + -- + -- Both aggregates order by `old_id`, which is what makes the two arrays + -- correspond positionally; `array_agg` without a matching ORDER BY would + -- be free to pair a file with another file's satellites. + IF v_files > 0 THEN + PERFORM storage.copy_file_satellites( + (SELECT array_agg(old_id ORDER BY old_id) FROM _copy_file_map), + (SELECT array_agg(new_id ORDER BY old_id) FROM _copy_file_map) + ); + END IF; + + -- Folder dead properties. Files are handled inside copy_file_satellites; + -- folders have no other satellites, so this stays here. + INSERT INTO storage.webdav_dead_properties + (folder_id, namespace, local_name, value) + SELECT cm.new_id, dp.namespace, dp.local_name, dp.value + FROM storage.webdav_dead_properties dp + JOIN _copy_map cm ON dp.folder_id = cm.old_id; + + RETURN QUERY SELECT v_new_root::text, v_folders, v_files; +END; +$$ LANGUAGE plpgsql; diff --git a/migrations/20261020000000_file_attached_blobs.sql b/migrations/20261020000000_file_attached_blobs.sql new file mode 100644 index 00000000..d8d1b346 --- /dev/null +++ b/migrations/20261020000000_file_attached_blobs.sql @@ -0,0 +1,157 @@ +-- Step 9 of `docs/plan/derived-blobs.md` — the file-keyed half of the pair. +-- +-- `content_derived_blobs` holds bytes that are a pure deterministic function +-- of a file's content, so they are keyed by that content and shared across +-- every file holding it. This table holds the opposite: bytes a USER supplied +-- or chose. Those must never be shared across files, and the key is what +-- enforces it. +-- +-- The distinction is a security boundary, not a modelling preference. If a +-- client-uploaded preview were content-keyed, user A could upload a file plus +-- a preview that misrepresents it; when user B later uploads the same bytes, +-- dedup would match and B would be served A's preview. Content-keying is only +-- safe when the bytes are derivable from the content by the server — nothing +-- to poison, because anyone with the same input gets the same output. +-- +-- Required now rather than deferred: the SPA already generates and PUTs +-- previews for PDFs, and there is no server-side regeneration path for them, +-- so the sidecar migration has nowhere else to put those bytes. + +CREATE TABLE IF NOT EXISTS storage.file_attached_blobs ( + file_id UUID NOT NULL REFERENCES storage.files(id) ON DELETE CASCADE, + kind TEXT NOT NULL CHECK (kind IN ('preview', 'subtitle', 'cover_art')), + variant TEXT NOT NULL, + blob_hash VARCHAR(64) NOT NULL, + content_type TEXT NOT NULL, + -- Provenance convention: NOT NULL and NO foreign key. A FK with + -- ON DELETE SET NULL loses the audit trail exactly when it matters most, + -- and without an ON DELETE clause it would block deleting a user + -- outright. Deleting the uploader must not rewrite history, so the id is + -- retained even once it no longer resolves. Rows imported with no known + -- uploader carry the all-zeros sentinel. + uploaded_by UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (file_id, kind, variant) +); + +-- Reverse lookup for the reference recompute and for dedup_gc's reap +-- predicate: both ask "does any row reference this blob?". +CREATE INDEX IF NOT EXISTS idx_file_attached_blobs_blob_hash + ON storage.file_attached_blobs (blob_hash); + +-- ── The routing rule, recorded on both tables ──────────────────────────── +-- Choosing the wrong table is a silent poisoning bug rather than a compile +-- error, so the rule lives where an implementor will actually meet it. + +COMMENT ON TABLE storage.file_attached_blobs IS + 'User-supplied or user-chosen artifacts (client previews, subtitles, cover art) keyed by FILE. File-keyed on purpose: these bytes are not derivable from the file''s content, so sharing them across files with identical content would let one user''s upload be served for another user''s file. Server-derived bytes must NOT be stored here — see docs/plan/derived-blobs.md.'; + +COMMENT ON COLUMN storage.file_attached_blobs.file_id IS + 'The file these bytes are attached to. ON DELETE CASCADE: the attachment has no meaning without it. Deleting the row does NOT release the blob reference — the owning service does that in its on_file_deleted hook.'; + +COMMENT ON COLUMN storage.file_attached_blobs.blob_hash IS + 'The attached Blob. Reference HOLDER — bumps chunk_manifests.ref_count via DedupService::add_reference. Dedup still applies to the bytes themselves; what is forbidden is sharing the MAPPING across files.'; + +COMMENT ON COLUMN storage.file_attached_blobs.variant IS + 'Opaque discriminator within a kind (preview | en | fr | cover...). New axes go inside this string, never into new columns.'; + +COMMENT ON COLUMN storage.file_attached_blobs.uploaded_by IS + 'Who supplied these bytes. Retained after the user is deleted — deleting a user must not rewrite provenance. The only trace that an Editor on a shared file replaced the owner''s preview.'; + +COMMENT ON TABLE storage.content_derived_blobs IS + 'Server-derived artifacts (thumbnails, transcodes) keyed by the BLAKE3 of their SOURCE content. Content-keyed on purpose: identical content shares one derivation. ROUTING RULE — bytes that are a pure deterministic function of the file''s content belong here; bytes that are user-supplied or user-chosen belong in storage.file_attached_blobs, which is file-keyed and never shared. See docs/plan/derived-blobs.md.'; + +-- ── Teach the copy fan-out about it ────────────────────────────────────── +-- +-- Only the attached-blobs arm is new versus `20261019000000`; everything +-- else is that definition verbatim. Adding a file-keyed table is now one +-- edit in one function, which is the whole point of having consolidated the +-- two copy paths first. +CREATE OR REPLACE FUNCTION storage.copy_file_satellites( + p_old_ids UUID[], + p_new_ids UUID[] +) RETURNS void AS $$ +DECLARE + v_unmatched TEXT[]; +BEGIN + IF p_old_ids IS NULL OR cardinality(p_old_ids) = 0 THEN + RETURN; + END IF; + + IF p_new_ids IS NULL OR cardinality(p_old_ids) <> cardinality(p_new_ids) THEN + -- Positional correspondence is the whole interface; a length + -- mismatch would silently attach satellites to the wrong file. + RAISE EXCEPTION + 'copy_file_satellites: id arrays must correspond positionally (% old vs % new)', + cardinality(p_old_ids), COALESCE(cardinality(p_new_ids), 0); + END IF; + + -- 1. WebDAV dead properties. RFC 4918 §8.8 requires COPY to duplicate + -- them: properties describe the resource, and the copy is a resource. + INSERT INTO storage.webdav_dead_properties + (file_id, namespace, local_name, value) + SELECT m.new_id, dp.namespace, dp.local_name, dp.value + FROM unnest(p_old_ids, p_new_ids) AS m(old_id, new_id) + JOIN storage.webdav_dead_properties dp ON dp.file_id = m.old_id; + + -- 2. A reference on the copied content, so deleting the original cannot + -- reap bytes the copy still needs. Read from the NEW rows rather than + -- the old ones: that is what makes an unreferenceable copy impossible + -- to create, since a row that failed to insert contributes nothing. + SELECT storage.add_blob_references(array_agg(f.blob_hash)) + INTO v_unmatched + FROM unnest(p_new_ids) AS n(id) + JOIN storage.files f ON f.id = n.id + WHERE NOT f.is_trashed; + + IF v_unmatched IS NOT NULL AND cardinality(v_unmatched) > 0 THEN + -- Warn, do not abort. A missing registry row means the SOURCE file + -- was already broken; the copy merely inherits it. Failing here + -- would abort an entire folder copy over one pre-existing fault, + -- which is worse than completing it and reporting. The blob-level + -- audit jobs are what surface the underlying breakage. + RAISE WARNING + 'copy_file_satellites: % copied file(s) reference a blob with no registry row (first: %); source was already broken', + cardinality(v_unmatched), v_unmatched[1]; + END IF; + + -- 3. File-keyed attachments — client previews, subtitles, cover art. + -- DUPLICATED rather than shared, because the key is `file_id` and the + -- copy is a different file. `uploaded_by` carries over: the person + -- who supplied the bytes did not change because someone copied the + -- file, and rewriting it to the copier would forge provenance. + -- + -- Each duplicated row is a new reference on the same blob, so the + -- bytes are still deduplicated — it is the MAPPING that must not be + -- shared, not the content. + WITH copied AS ( + INSERT INTO storage.file_attached_blobs + (file_id, kind, variant, blob_hash, content_type, uploaded_by) + SELECT m.new_id, a.kind, a.variant, a.blob_hash, a.content_type, a.uploaded_by + FROM unnest(p_old_ids, p_new_ids) AS m(old_id, new_id) + JOIN storage.file_attached_blobs a ON a.file_id = m.old_id + RETURNING blob_hash + ) + SELECT storage.add_blob_references(array_agg(blob_hash)) + INTO v_unmatched + FROM copied; + + IF v_unmatched IS NOT NULL AND cardinality(v_unmatched) > 0 THEN + RAISE WARNING + 'copy_file_satellites: % attached blob(s) reference no registry row (first: %); source was already broken', + cardinality(v_unmatched), v_unmatched[1]; + END IF; + + -- ── Deliberately absent ────────────────────────────────────────────── + -- + -- storage.comments (future): NOT copied. A copy is a new artifact; the + -- discussion belongs to the original. + -- + -- content_derived_blobs, blob_extracted_text, faces.faces: content-keyed. + -- The copy shares the source's hash, so it already sees them — copying + -- would duplicate rows that are keyed on the very thing being shared. + -- + -- storage.favorites, recent_items, shares: properties of the ORIGINAL's + -- relationship to users, not of its content. +END; +$$ LANGUAGE plpgsql; diff --git a/migrations/20261021000000_file_attached_blobs_decrement_trigger.sql b/migrations/20261021000000_file_attached_blobs_decrement_trigger.sql new file mode 100644 index 00000000..a1d8c368 --- /dev/null +++ b/migrations/20261021000000_file_attached_blobs_decrement_trigger.sql @@ -0,0 +1,28 @@ +-- Release the blob reference when an attachment row goes away. +-- +-- `storage.file_attached_blobs.file_id` is `ON DELETE CASCADE`, so deleting a +-- file removes its attachment rows inside the database — invisible to Rust. +-- The lifecycle hook cannot cover this: `on_file_deleted` fires AFTER +-- `delete_file`, by which point the cascade has already run and there is +-- nothing left to read. The references would survive with no row behind them, +-- and `dedup_gc` would see a positive count forever — bytes pinned for good. +-- +-- `storage.decrement_blob_ref()` already exists for exactly this, on +-- `storage.files`. It keys off `OLD.blob_hash` and is otherwise +-- table-agnostic, so it applies verbatim — and reusing it keeps the +-- manifest-first decrement contract defined in one place rather than +-- transcribed into a second trigger that can drift. +-- +-- Only DELETE. Replacing a preview updates `blob_hash` in place +-- (`store_attached_blob` is ON CONFLICT DO UPDATE), and the reference to the +-- superseded blob is released there, in Rust. Adding UPDATE here would +-- double-decrement it. + +CREATE OR REPLACE TRIGGER trg_file_attached_blobs_decrement_blob_ref + AFTER DELETE ON storage.file_attached_blobs + FOR EACH ROW + EXECUTE FUNCTION storage.decrement_blob_ref(); + +COMMENT ON TRIGGER trg_file_attached_blobs_decrement_blob_ref + ON storage.file_attached_blobs IS + 'Releases the blob reference held by an attachment row. Needed because file_id is ON DELETE CASCADE, so rows vanish inside the DB where the Rust lifecycle hooks cannot see them.'; diff --git a/migrations/20261022000000_derived_variant_encodes_format.sql b/migrations/20261022000000_derived_variant_encodes_format.sql new file mode 100644 index 00000000..f3efcd16 --- /dev/null +++ b/migrations/20261022000000_derived_variant_encodes_format.sql @@ -0,0 +1,56 @@ +-- Put the output format inside `variant`, where the plan says new axes go. +-- +-- `content_derived_blobs.variant` held the size alone (`icon` | `preview` | +-- `large`), so a size could hold exactly ONE stored artifact regardless of +-- codec. That surfaced when the read order flipped (step 10c): a JPEG request +-- matched the WebP row and would have been served the wrong codec, which the +-- old ordering hid because the `.jpg` sidecar won first. The flip had to be +-- gated to WebP, which in turn means JPEG clients can never leave the sidecar +-- — so the sidecar can never be deleted. +-- +-- It blocks transcodes harder still: those are multi-format by nature, so +-- without a format term two output codecs of one source collide on the +-- primary key. +-- +-- Per the column's own comment — "new axes go inside this string, never into +-- new columns" — the axis goes in the string rather than into a fourth PK +-- column. The PK stays `(source_hash, kind, variant)`. +-- +-- Shape: `{size}.{ext}` — `preview.webp`, `icon.jpg`, and later `720p.webp` +-- for transcodes. +-- +-- The backfill is deterministic rather than a guess: `store_derived_blob` has +-- only ever been called with `"image/webp"` for thumbnails, so every existing +-- thumbnail row is WebP. `content_type` is checked anyway rather than assumed +-- — if that assumption is ever wrong, the row is left alone for a human to +-- look at instead of being silently mislabelled. + +UPDATE storage.content_derived_blobs + SET variant = variant || '.webp' + WHERE kind = 'thumbnail' + AND content_type = 'image/webp' + -- Idempotent: skip anything already carrying a format suffix, so a + -- re-applied migration cannot produce `preview.webp.webp`. + AND variant NOT LIKE '%.%'; + +-- Anything left without a format suffix did not match the WebP assumption. +-- Surfaced as a warning rather than coerced: the read path will simply miss +-- those rows and fall back to the sidecar, which is safe, whereas guessing a +-- codec would serve the wrong bytes. +DO $$ +DECLARE + v_unsuffixed INT; +BEGIN + SELECT COUNT(*) INTO v_unsuffixed + FROM storage.content_derived_blobs + WHERE kind = 'thumbnail' AND variant NOT LIKE '%.%'; + + IF v_unsuffixed > 0 THEN + RAISE WARNING + 'derived_variant_encodes_format: % thumbnail row(s) have no format suffix (content_type was not image/webp). They will be ignored by the read path and re-derived on demand; inspect before deleting the sidecars.', + v_unsuffixed; + END IF; +END $$; + +COMMENT ON COLUMN storage.content_derived_blobs.variant IS + 'Opaque discriminator carrying every axis but the source and the kind: size AND output format, as {size}.{ext} (preview.webp | icon.jpg | 720p.webp). New axes go inside this string, never into new columns. A format term is required — without one, two codecs of the same source collide on the primary key, and the read path cannot tell which codec a row holds.'; diff --git a/migrations/20261023000000_content_derived_blobs_negative_rows.sql b/migrations/20261023000000_content_derived_blobs_negative_rows.sql new file mode 100644 index 00000000..b119ade7 --- /dev/null +++ b/migrations/20261023000000_content_derived_blobs_negative_rows.sql @@ -0,0 +1,63 @@ +-- Negative rows in `storage.content_derived_blobs`. +-- +-- Some derivations can only be known to be useless by doing the whole +-- expensive job. `ImageTranscodeService` learns that WebP is not smaller +-- than the original by decoding and re-encoding the whole image; a +-- thumbnail renderer learns a source is undecodable, or over the +-- 50-megapixel ceiling, only by attempting it. Recomputing that verdict +-- on every request is the same cost as computing it the first time. +-- +-- Today those verdicts live in RAM (moka's zero-weight empty-Bytes +-- convention) and, for transcodes, as zero-byte `.skip` files on local +-- disk. Both vanish: moka evicts, and the local disk is exactly what +-- this plan is deleting. So the verdict is stored here, next to the +-- positive derivations, as a row whose derived Blob is NULL. +-- +-- ## Why NULL rather than a sentinel hash +-- +-- A reserved hash was considered and rejected. It would stop +-- `blob_hash` naming a real Blob, and every consumer — the refcount +-- recompute in `manifests_consistency`, `dedup_gc`'s reap predicate, +-- `satellites_consistency`'s dangling check — would need to learn the +-- exception or silently mis-handle it. NULL is already the SQL way to +-- say "no Blob", and those consumers all join on `blob_hash`, so a NULL +-- drops out of the join instead of matching something fictional. +-- +-- ## The CHECK matters +-- +-- A row with a `blob_hash` but no `content_type` is unserveable; a row +-- with a `content_type` but no `blob_hash` claims a type for bytes that +-- do not exist. Both are bugs that would surface far from their cause, +-- so the pair moves together or not at all. +-- +-- ## What must NOT become a negative row +-- +-- Only failures that are DETERMINISTIC IN THE CONTENT. A transcode that +-- was not smaller, or a source that cannot be decoded, will fail the +-- same way forever — those are worth remembering. A generation timeout, +-- a closed semaphore, an I/O error reading the source Blob are +-- properties of the moment, not the content; persisting one marks a +-- perfectly good image as underivable permanently, and nothing ever +-- retries it. The asymmetry sets the default: a wrongly-cached +-- transient is silent and forever, a missing negative merely costs +-- repeated work. When in doubt, do not write the row. + +ALTER TABLE storage.content_derived_blobs + ALTER COLUMN blob_hash DROP NOT NULL, + ALTER COLUMN content_type DROP NOT NULL; + +ALTER TABLE storage.content_derived_blobs + DROP CONSTRAINT IF EXISTS content_derived_blobs_positive_or_negative; + +ALTER TABLE storage.content_derived_blobs + ADD CONSTRAINT content_derived_blobs_positive_or_negative + CHECK ( + (blob_hash IS NOT NULL AND content_type IS NOT NULL) + OR (blob_hash IS NULL AND content_type IS NULL) + ); + +COMMENT ON COLUMN storage.content_derived_blobs.blob_hash IS + 'The derived Blob, or NULL for a NEGATIVE row: the derivation was attempted and is known not to be worth storing (transcode came out larger, source undecodable, source over the decode ceiling). Reference HOLDER when present — bumps chunk_manifests.ref_count via DedupService::add_reference. Only content-deterministic failures may be recorded as negatives; transient ones (timeout, semaphore, I/O) must not, or a momentary failure becomes permanent.'; + +COMMENT ON COLUMN storage.content_derived_blobs.content_type IS + 'MIME type of the derived Blob. NULL exactly when blob_hash is NULL — the CHECK keeps the pair together, since a type without bytes describes nothing and bytes without a type cannot be served.'; diff --git a/migrations/20261024000000_nfc_name_column_comments.sql b/migrations/20261024000000_nfc_name_column_comments.sql new file mode 100644 index 00000000..6b4055d7 --- /dev/null +++ b/migrations/20261024000000_nfc_name_column_comments.sql @@ -0,0 +1,52 @@ +-- COMMENT ON COLUMN for every user-visible-name column whose invariant +-- ("stored bytes are NFC") is enforced by the write repository, not by +-- the DB itself. +-- +-- Why this migration exists +-- ───────────────────────── +-- Before 2026-09-04 the NFC invariant lived at `File::new` / +-- `Folder::new_folder` entity constructors — plausible-looking but +-- DEAD CODE for the create path, because every real caller went +-- straight from a DTO string to `sqlx::bind()` inside the repos +-- without ever constructing the entity first. Result: 22 audited +-- entry points, every single one shipped raw client input to the DB. +-- macOS Finder / DAVX5 / NC-desktop uploads landed NFD; NFC- +-- normalizing clients then failed to find their own content by URL +-- (AtalayaLabs/OxiCloud#706). +-- +-- The fix moved normalization to the repository methods that own the +-- INSERT / UPDATE. The next contributor writing a new write surface +-- may reasonably wonder where to enforce the invariant — this comment +-- puts the answer next to the column so grep-hunting the codebase is +-- not required. Purely documentation; no runtime effect. A stronger +-- form (CHECK CONSTRAINT `name = normalize(name, NFC)`) was +-- considered and rejected for now — that would rely on every +-- historical row already being NFC (which we deliberately do NOT +-- migrate on read, so pre-fix rows stay in place until an operator +-- runs `oxicloud migrate nfc-filenames`), and would fail-boot any +-- upgrade path where the migrate has not yet been applied. +-- +-- Idempotent. COMMENT ON COLUMN replaces any prior comment on the +-- same target, so re-running has no effect. + +COMMENT ON COLUMN storage.files.name IS + 'User-visible file name. MUST be NFC (Unicode Normalization Form C). ' + 'Invariant enforced at write time by ' + 'src/infrastructure/repositories/pg/file_blob_write_repository.rs — the ' + '`save_file_with_blob_impl`, `copy_file`, `rename_file`, ' + '`register_file_deferred`, and `copy_folder_tree` methods each call ' + '`normalize_storage_name(_owned)` before binding. No DB-level CHECK ' + 'constraint (historical NFD rows may still exist on pre-2026-09-04 ' + 'databases until `oxicloud migrate nfc-filenames` is run). New write ' + 'surfaces MUST land in one of those repo methods; direct INSERT ' + 'bypasses the invariant.'; + +COMMENT ON COLUMN storage.folders.name IS + 'User-visible folder name. MUST be NFC (Unicode Normalization Form C). ' + 'Invariant enforced at write time by ' + 'src/infrastructure/repositories/pg/folder_db_repository.rs — the ' + '`create_folder` and `rename_folder` methods each call ' + '`normalize_storage_name_owned` before binding. See also ' + 'storage.files.name — identical contract, different table. No DB-level ' + 'CHECK (see that column comment). New write surfaces MUST land in one of ' + 'those repo methods; direct INSERT bypasses the invariant.'; diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 00000000..a82a9de0 --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +NOW="$(date '+%Y-%m-%d %H:%M:%S')" + +pg_dump postgres://postgres:postgres@localhost:5432/oxicloud -F c --disable-triggers > "backup.${NOW}.dump" diff --git a/scripts/compute-docker-tags.sh b/scripts/compute-docker-tags.sh new file mode 100755 index 00000000..765ef171 --- /dev/null +++ b/scripts/compute-docker-tags.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# ============================================================= +# Compute Docker channel + version + tag set for the +# `docker-publish` workflow. Extracted from the workflow so the +# logic can be unit-tested via `scripts/test-docker-publish-tags.sh` +# without needing to dispatch the workflow itself. +# +# The workflow's meta step invokes this via `bash scripts/ +# compute-docker-tags.sh` with the GITHUB_* env vars set; the +# same call form works from a local shell for smoke checks +# ("what would we publish if I tag v0.8.8 tomorrow?"). +# +# Inputs (env vars — missing required inputs exit non-zero): +# EVENT_NAME workflow_dispatch | release | push +# GITHUB_REF refs/heads/main | refs/tags/vX.Y.Z | ... +# (required for the `push` event) +# DISPATCH_VERSION workflow_dispatch only, e.g. v0.5.3 or 0.5.3 +# RELEASE_TAG release event only, e.g. v0.8.7 +# REGISTRY_IMAGE Docker Hub image (e.g. diocrafts/oxicloud) +# GHCR_REGISTRY_IMAGE GHCR image (e.g. ghcr.io/atalayalabs/oxicloud) +# SKIP_DOCKERHUB optional. When "true", omits Docker Hub tags +# from the output — used by forks whose +# DOCKERHUB_TOKEN secret isn't configured. The +# workflow only pushes to GHCR (which needs no +# external secret; auth via GITHUB_TOKEN). +# +# Outputs: +# Always writes `version=`, `channel=`, and a `tags:` block +# to stdout — visible in workflow logs and captured by the test +# harness for diff-based assertions. +# +# When `GITHUB_OUTPUT` is set (inside a GHA `run:` step), also +# emits the same values via GHA's `>> $GITHUB_OUTPUT` convention +# so subsequent steps can reference `${{ steps.meta.outputs.tags }}`. +# +# Channel semantics (mirrors the workflow's tag policy): +# release — tag push / release event / manual dispatch: +# publish `:` AND move `:latest`. +# main — push to `main` branch: publish `:main` (mutable +# tip) only. Never touches `:latest`, never emits a +# per-commit `:main-` (would balloon the +# registry across every merge). +# ============================================================= + +set -euo pipefail + +: "${EVENT_NAME:?EVENT_NAME required}" +: "${REGISTRY_IMAGE:?REGISTRY_IMAGE required}" +: "${GHCR_REGISTRY_IMAGE:?GHCR_REGISTRY_IMAGE required}" + +# Both GHCR and Docker Hub reject mixed-case namespace / image names +# ("repository name must be lowercase"). `${{ github.repository_owner +# }}` in the workflow inserts the GitHub username verbatim, and GitHub +# expression syntax has no `lower()` function. So we normalise here +# — the workflow keeps its declarative `env:` block, the script owns +# the case-safety contract, and tests cover it (see +# `test-docker-publish-tags.sh` for mixed-case cases). +REGISTRY_IMAGE=$(echo "$REGISTRY_IMAGE" | tr '[:upper:]' '[:lower:]') +GHCR_REGISTRY_IMAGE=$(echo "$GHCR_REGISTRY_IMAGE" | tr '[:upper:]' '[:lower:]') + +# Docker Hub is optional — a fork without DOCKERHUB_TOKEN configured +# still publishes to GHCR (its own namespace, auth via GITHUB_TOKEN) +# but skips Docker Hub cleanly. The workflow sets this to "true" when +# `secrets.DOCKERHUB_TOKEN` is empty; the tag set below omits DH +# entries in that case. +SKIP_DOCKERHUB="${SKIP_DOCKERHUB:-false}" + +case "$EVENT_NAME" in + workflow_dispatch) + : "${DISPATCH_VERSION:?DISPATCH_VERSION required for workflow_dispatch}" + VERSION="${DISPATCH_VERSION#v}" + CHANNEL="release" + ;; + release) + : "${RELEASE_TAG:?RELEASE_TAG required for release event}" + VERSION="${RELEASE_TAG#v}" + CHANNEL="release" + ;; + push) + : "${GITHUB_REF:?GITHUB_REF required for push event}" + if [ "$GITHUB_REF" = "refs/heads/main" ]; then + VERSION="main" + CHANNEL="main" + else + # Tag push — strip refs/tags/ prefix and leading `v`. + VERSION="${GITHUB_REF#refs/tags/}" + VERSION="${VERSION#v}" + CHANNEL="release" + fi + ;; + *) + echo "compute-docker-tags: unsupported EVENT_NAME: $EVENT_NAME" >&2 + exit 1 + ;; +esac + +# Assemble the multi-line tag set. Format matches what the +# `docker/build-push-action` `tags:` input consumes — one tag +# per line, whitespace ignored between lines. Docker Hub entries +# omitted entirely when SKIP_DOCKERHUB=true — the build-push-action +# just doesn't see the tags, so no auth is attempted for them. +if [ "$CHANNEL" = "main" ]; then + if [ "$SKIP_DOCKERHUB" = "true" ]; then + TAGS="${GHCR_REGISTRY_IMAGE}:main" + else + TAGS="${REGISTRY_IMAGE}:main +${GHCR_REGISTRY_IMAGE}:main" + fi +else + if [ "$SKIP_DOCKERHUB" = "true" ]; then + TAGS="${GHCR_REGISTRY_IMAGE}:${VERSION} +${GHCR_REGISTRY_IMAGE}:latest" + else + TAGS="${REGISTRY_IMAGE}:${VERSION} +${REGISTRY_IMAGE}:latest +${GHCR_REGISTRY_IMAGE}:${VERSION} +${GHCR_REGISTRY_IMAGE}:latest" + fi +fi + +# Emit to $GITHUB_OUTPUT when running under GHA — subsequent +# workflow steps read via `${{ steps.meta.outputs.tags }}`. +if [ -n "${GITHUB_OUTPUT:-}" ]; then + { + echo "version=$VERSION" + echo "channel=$CHANNEL" + echo "tags<> "$GITHUB_OUTPUT" +fi + +# Also emit VERSION + CHANNEL + SKIP_DOCKERHUB to $GITHUB_ENV — +# later steps (`Verify published image`, DockerHub-gated conditionals) +# read these directly. Keeps the step-scoped env aligned with the +# steps.meta.outputs.* set for consumers that prefer one or the other. +# +# Also OVERRIDE the workflow-level REGISTRY_IMAGE / GHCR_REGISTRY_IMAGE +# env vars with the lowercased forms. Without this, the verify step +# would read the workflow-declared mixed-case value from +# `${{ github.repository_owner }}` (e.g. `ghcr.io/EdouardVanbelle/ +# oxicloud`) and `docker pull` would reject it — despite the tags +# themselves being lowercased in the actual push. Step-level env +# additions take precedence over workflow-level for subsequent steps. +if [ -n "${GITHUB_ENV:-}" ]; then + { + echo "VERSION=$VERSION" + echo "CHANNEL=$CHANNEL" + echo "SKIP_DOCKERHUB=$SKIP_DOCKERHUB" + echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" + echo "GHCR_REGISTRY_IMAGE=$GHCR_REGISTRY_IMAGE" + } >> "$GITHUB_ENV" +fi + +# Always echo to stdout — visible in workflow logs (useful for +# dry-run verification, when the push step is skipped) and +# consumed by the test harness for equality checks. +echo "version=$VERSION" +echo "channel=$CHANNEL" +echo "tags:" +echo "$TAGS" | sed 's/^/ /' diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..6915c744 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# scripts/restore.sh — restore a pg_dump custom-format snapshot into a +# fresh oxicloud DB. +# +# Drops + recreates the whole DB before restoring so migrations added +# AFTER the dump was taken don't block --clean's DROPs. Symptom of +# that class (hit 2026-08-30): +# +# pg_restore: erreur : ... +# cannot drop constraint files_pkey on table storage.files because +# other objects depend on it +# DÉTAIL : constraint file_attached_blobs_file_id_fkey on table +# storage.file_attached_blobs depends on index storage.files_pkey +# +# The dump only knows about objects that existed at dump time; +# `pg_restore --clean` DROPs exactly those. Anything added since +# (`file_attached_blobs` in the example above) survives and blocks +# DROPs of things it depends on. Drop-and-recreate the whole DB +# sidesteps the problem entirely. +# +# Usage: +# ./scripts/restore.sh +# +# Companion of the pg_dump command in memory +# bug_pg_dump_folders_circular_fk.md: +# pg_dump postgres://... -F c --disable-triggers > backup.$NOW.dump +# +# NB: this restores the DB ONLY. Blob storage on disk +# (${OXICLOUD_STORAGE_PATH}) is NOT touched — snapshot + restore that +# separately with rsync if you need lockstep DB/disk state. + +set -euo pipefail + +DUMP="${1:?usage: $0 }" +[[ -f "$DUMP" ]] || { echo "[restore] ERROR: dump file not found: $DUMP" >&2; exit 1; } + +# Admin connection — connect to the `postgres` maintenance DB so we can +# drop `oxicloud` itself (can't drop the DB you're connected to). Both +# connection strings share credentials from the sandbox setup. +ADMIN="postgres://postgres:postgres@localhost:5432/postgres" +TARGET="postgres://postgres:postgres@localhost:5432/oxicloud" + +# 1. Terminate every connection to `oxicloud` so DROP DATABASE can +# proceed. OxiCloud running against 5432? rust-analyzer with +# sqlx-cli open? Any lingering psql session? All of them block +# `DROP DATABASE` with "database is being accessed by other users". +# pg_terminate_backend kicks them cleanly (they'll reconnect if +# they retry). +echo "[restore] Terminating connections to oxicloud..." +psql "$ADMIN" -c " + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = 'oxicloud' + AND pid <> pg_backend_pid(); +" >/dev/null + +# 2. Drop + recreate. IF EXISTS on DROP so a fresh workstation without +# an existing `oxicloud` database doesn't error on the first-ever +# invocation. +echo "[restore] Dropping + recreating oxicloud database..." +psql "$ADMIN" <<'SQL' +DROP DATABASE IF EXISTS oxicloud; +CREATE DATABASE oxicloud OWNER postgres; +SQL + +# 3. Restore. `--clean --if-exists` no longer needed (fresh DB from +# step 2). The remaining flags: +# +# * --disable-triggers — turn triggers off during data load so the +# `storage.folders.parent_id` self-FK doesn't reject rows whose +# parent hasn't been inserted yet in the same COPY batch. See +# memory bug_pg_dump_folders_circular_fk for background. +# * --single-transaction — atomic restore (all-or-nothing) AND +# lets --disable-triggers work without superuser (superuser is +# required otherwise). +# * --no-owner --no-privileges — portable, ignores ownership / +# GRANTs from the source system so a dump taken from one machine +# restores cleanly on another with different user names. +echo "[restore] Restoring from $DUMP..." +pg_restore \ + --disable-triggers \ + --single-transaction \ + --no-owner \ + --no-privileges \ + -d "$TARGET" \ + "$DUMP" + +echo "[restore] Done. Database restored to snapshot state in $DUMP." diff --git a/scripts/test-docker-publish-tags.sh b/scripts/test-docker-publish-tags.sh new file mode 100755 index 00000000..ad38bacf --- /dev/null +++ b/scripts/test-docker-publish-tags.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +# ============================================================= +# Unit tests for `scripts/compute-docker-tags.sh`. +# +# Exercises every trigger channel the docker-publish workflow +# supports, plus the invalid-input path. Runs standalone in +# under a second — cheap regression check to lock the tag-set +# contract before pushing changes to `.github/workflows/ +# docker-publish.yml`. +# +# Run: +# bash scripts/test-docker-publish-tags.sh +# ============================================================= + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SCRIPT="$SCRIPT_DIR/compute-docker-tags.sh" + +if [ ! -f "$SCRIPT" ]; then + echo "compute-docker-tags.sh not found at $SCRIPT" >&2 + exit 2 +fi + +REGISTRY_IMAGE=diocrafts/oxicloud +GHCR_REGISTRY_IMAGE=ghcr.io/atalayalabs/oxicloud + +pass=0 +fail=0 + +# Runs the script with the given env, compares stdout against expected. +# `env "$@" bash ...` passes the env vars only for this invocation so +# leftover state from a prior case can't leak across. +expect() { + local name="$1" expected="$2" + shift 2 + local actual rc + actual=$(env -i \ + REGISTRY_IMAGE="$REGISTRY_IMAGE" \ + GHCR_REGISTRY_IMAGE="$GHCR_REGISTRY_IMAGE" \ + PATH="/usr/bin:/bin" \ + "$@" \ + bash "$SCRIPT" 2>&1) + rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL: $name — script exited $rc:" + echo "$actual" | sed 's/^/ /' + fail=$((fail + 1)) + return + fi + if [ "$actual" = "$expected" ]; then + echo "PASS: $name" + pass=$((pass + 1)) + else + echo "FAIL: $name" + echo " expected:" + echo "$expected" | sed 's/^/ /' + echo " actual:" + echo "$actual" | sed 's/^/ /' + fail=$((fail + 1)) + fi +} + +expect_fail() { + local name="$1" + shift + if env -i \ + REGISTRY_IMAGE="$REGISTRY_IMAGE" \ + GHCR_REGISTRY_IMAGE="$GHCR_REGISTRY_IMAGE" \ + PATH="/usr/bin:/bin" \ + "$@" \ + bash "$SCRIPT" >/dev/null 2>&1 + then + echo "FAIL: $name (script should have exited non-zero)" + fail=$((fail + 1)) + else + echo "PASS: $name" + pass=$((pass + 1)) + fi +} + +# ── Happy paths ───────────────────────────────────────────────── + +expect "push to main → :main only, no :latest" \ +"version=main +channel=main +tags: + diocrafts/oxicloud:main + ghcr.io/atalayalabs/oxicloud:main" \ + EVENT_NAME=push GITHUB_REF=refs/heads/main + +expect "push of version tag → : + :latest" \ +"version=0.8.7 +channel=release +tags: + diocrafts/oxicloud:0.8.7 + diocrafts/oxicloud:latest + ghcr.io/atalayalabs/oxicloud:0.8.7 + ghcr.io/atalayalabs/oxicloud:latest" \ + EVENT_NAME=push GITHUB_REF=refs/tags/v0.8.7 + +expect "release event → : + :latest" \ +"version=0.9.0 +channel=release +tags: + diocrafts/oxicloud:0.9.0 + diocrafts/oxicloud:latest + ghcr.io/atalayalabs/oxicloud:0.9.0 + ghcr.io/atalayalabs/oxicloud:latest" \ + EVENT_NAME=release RELEASE_TAG=v0.9.0 + +expect "workflow_dispatch with 'v' prefix" \ +"version=1.0.0 +channel=release +tags: + diocrafts/oxicloud:1.0.0 + diocrafts/oxicloud:latest + ghcr.io/atalayalabs/oxicloud:1.0.0 + ghcr.io/atalayalabs/oxicloud:latest" \ + EVENT_NAME=workflow_dispatch DISPATCH_VERSION=v1.0.0 + +expect "workflow_dispatch without 'v' prefix (permissive)" \ +"version=1.0.0 +channel=release +tags: + diocrafts/oxicloud:1.0.0 + diocrafts/oxicloud:latest + ghcr.io/atalayalabs/oxicloud:1.0.0 + ghcr.io/atalayalabs/oxicloud:latest" \ + EVENT_NAME=workflow_dispatch DISPATCH_VERSION=1.0.0 + +expect "release with pre-release version" \ +"version=0.9.0-rc1 +channel=release +tags: + diocrafts/oxicloud:0.9.0-rc1 + diocrafts/oxicloud:latest + ghcr.io/atalayalabs/oxicloud:0.9.0-rc1 + ghcr.io/atalayalabs/oxicloud:latest" \ + EVENT_NAME=release RELEASE_TAG=v0.9.0-rc1 + +# ── SKIP_DOCKERHUB path (forks without DOCKERHUB_TOKEN) ───────── + +expect "push to main + SKIP_DOCKERHUB → GHCR only" \ +"version=main +channel=main +tags: + ghcr.io/atalayalabs/oxicloud:main" \ + EVENT_NAME=push GITHUB_REF=refs/heads/main SKIP_DOCKERHUB=true + +expect "release + SKIP_DOCKERHUB → GHCR : + :latest only" \ +"version=0.9.0 +channel=release +tags: + ghcr.io/atalayalabs/oxicloud:0.9.0 + ghcr.io/atalayalabs/oxicloud:latest" \ + EVENT_NAME=release RELEASE_TAG=v0.9.0 SKIP_DOCKERHUB=true + +expect "dispatch + SKIP_DOCKERHUB → GHCR : + :latest only" \ +"version=1.0.0 +channel=release +tags: + ghcr.io/atalayalabs/oxicloud:1.0.0 + ghcr.io/atalayalabs/oxicloud:latest" \ + EVENT_NAME=workflow_dispatch DISPATCH_VERSION=v1.0.0 SKIP_DOCKERHUB=true + +expect "explicit SKIP_DOCKERHUB=false behaves like default (both registries)" \ +"version=main +channel=main +tags: + diocrafts/oxicloud:main + ghcr.io/atalayalabs/oxicloud:main" \ + EVENT_NAME=push GITHUB_REF=refs/heads/main SKIP_DOCKERHUB=false + +# ── Case-safety — GHCR / DH reject mixed-case names ───────────── +# +# Regression pin: `${{ github.repository_owner }}` inserts a +# GitHub username verbatim, which is often mixed-case +# (e.g. EdouardVanbelle). The registries reject that with +# "repository name must be lowercase". The script normalises +# both inputs; these cases assert it. + +# Local override so we can pass a mixed-case owner without touching +# the harness's defaults on other cases. +_orig_ghcr="$GHCR_REGISTRY_IMAGE" +_orig_dh="$REGISTRY_IMAGE" + +GHCR_REGISTRY_IMAGE=ghcr.io/EdouardVanbelle/OxiCloud \ +REGISTRY_IMAGE=ghcr.io/EdouardVanbelle/OxiCloud \ +expect "mixed-case owner and image lowercased in tags" \ +"version=main +channel=main +tags: + ghcr.io/edouardvanbelle/oxicloud:main" \ + EVENT_NAME=push GITHUB_REF=refs/heads/main \ + REGISTRY_IMAGE=DioCrafts/OxiCloud \ + GHCR_REGISTRY_IMAGE=ghcr.io/EdouardVanbelle/OxiCloud \ + SKIP_DOCKERHUB=true + +expect "release with mixed-case DH namespace lowercased" \ +"version=0.8.7 +channel=release +tags: + diocrafts/oxicloud:0.8.7 + diocrafts/oxicloud:latest + ghcr.io/edouardvanbelle/oxicloud:0.8.7 + ghcr.io/edouardvanbelle/oxicloud:latest" \ + EVENT_NAME=release RELEASE_TAG=v0.8.7 \ + REGISTRY_IMAGE=DioCrafts/OxiCloud \ + GHCR_REGISTRY_IMAGE=ghcr.io/EdouardVanbelle/OxiCloud + +REGISTRY_IMAGE="$_orig_dh" +GHCR_REGISTRY_IMAGE="$_orig_ghcr" +unset _orig_dh _orig_ghcr + +# ── Error paths ───────────────────────────────────────────────── + +expect_fail "unknown event rejected" \ + EVENT_NAME=cron GITHUB_REF=refs/heads/main + +expect_fail "push without GITHUB_REF rejected" \ + EVENT_NAME=push + +expect_fail "release without RELEASE_TAG rejected" \ + EVENT_NAME=release + +expect_fail "dispatch without DISPATCH_VERSION rejected" \ + EVENT_NAME=workflow_dispatch + +# ── Report ────────────────────────────────────────────────────── + +echo "" +echo "─────────────────────────" +echo "Passed: $pass Failed: $fail" +[ "$fail" -eq 0 ] diff --git a/src/application/dtos/session_dto.rs b/src/application/dtos/session_dto.rs index 87fe23ba..170457ee 100644 --- a/src/application/dtos/session_dto.rs +++ b/src/application/dtos/session_dto.rs @@ -14,6 +14,8 @@ //! separate batch fetch (extra round-trip). Frontend cross-references //! `user_id` against its cached user list. +use std::time::Duration; + use chrono::{DateTime, Utc}; use serde::Serialize; use utoipa::ToSchema; @@ -21,6 +23,20 @@ use uuid::Uuid; use crate::domain::entities::session::{Session, SessionOrigin}; +/// The "recently seen" threshold that turns a session's +/// `last_seen_at` into a green-dot "Online" badge on the admin +/// sessions panel — AND the same window that drives the +/// `oxicloud_sessions_online[_users]` Prometheus gauges (see +/// `src/infrastructure/services/session_liveness_gauges.rs`). +/// The two MUST agree so the dashboard's per-row badge count +/// matches the gauge's aggregate — one source of truth here. +/// +/// 5 min feels responsive without over-fluctuating with +/// tab-open-then-close blips. Deliberately hardcoded, not an +/// env var — see `docs/plan/sessions.md` §"Config surface" for +/// the reasoning. +pub const ONLINE_WINDOW: Duration = Duration::from_secs(5 * 60); + /// Authenticated-caller context — the caller's identity + session- /// bound signals a service method might key off. Constructed at the /// handler boundary from `AuthUser` and passed through unchanged; @@ -53,6 +69,17 @@ pub struct SessionSummaryDto { pub user_id: Uuid, pub created_at: DateTime, pub expires_at: DateTime, + /// Wall-clock time this session was last observed serving an + /// authenticated request. Moved forward per request by the + /// in-process [`LastSeenTracker`](crate::infrastructure::services::last_seen_tracker) + /// via a batched UPDATE every 30 s — so this value trails the + /// true "last seen" by at most one flush interval on a running + /// server. On DB read it always converges after a graceful + /// shutdown flush. Distinct from `created_at`: that only moves + /// on session rotation (silent refresh), so its resolution is + /// capped at the access-token TTL. The admin table renders a + /// "last seen X ago" column off this field. + pub last_seen_at: DateTime, pub ip_address: Option, pub user_agent: Option, /// `true` iff the session is DPoP-bound. Rendered as a lock icon @@ -68,7 +95,24 @@ pub struct SessionSummaryDto { pub is_revoked: bool, /// Whether this row is currently usable — `!revoked && expires_at > now()`. /// Kept server-side so the SPA doesn't drift if the browser clock is off. + /// **Distinct from [`is_online`](Self::is_online)** — this is a + /// *lifecycle* signal (row still has authority), that one is a + /// *presence* signal (a request landed on it lately). pub is_active: bool, + /// Whether the session was actually observed serving a request in the + /// last [`ONLINE_WINDOW`] (5 min). Presence signal, orthogonal to + /// [`is_active`](Self::is_active): a session may be active-and-online + /// (green dot in the admin table), active-and-idle (no dot, "last + /// seen 12 min ago"), or non-active-and-offline (expired / revoked + /// rows are never online). Derived server-side against + /// [`ONLINE_WINDOW`] so the row-level badge stays consistent with + /// the `oxicloud_sessions_online[_users]` Prometheus aggregates. + /// + /// Guaranteed `false` for revoked / expired rows — those short- + /// circuit before the recency check so a revoked row that happened + /// to receive a request in its final second before revocation + /// doesn't confusingly render "Online" post-revocation. + pub is_online: bool, // NOTE: no `oidc_sid` / `oidc_sid_prefix` field. The IdP-emitted // sid identifies the row's upstream session and stays server-side // (used by Back-Channel Logout matching). Exposing even a prefix @@ -104,23 +148,40 @@ impl SessionSummaryDto { pub fn from_session(s: Session, caller_jkt: Option<&str>) -> Self { let is_revoked = s.is_revoked(); let is_expired = s.is_expired(); + let is_active = !is_revoked && !is_expired; let jkt = s.dpop_jkt().map(|s| s.to_owned()); let dpop_jkt_prefix = jkt.as_ref().map(|t| t.chars().take(8).collect::()); let is_current = match (jkt.as_deref(), caller_jkt) { (Some(row), Some(caller)) => row == caller, _ => false, }; + // Presence check gated on lifecycle — a revoked or expired + // row's `last_seen_at` may still be fresh (the last request + // that arrived just before revocation), but calling it + // "Online" post-revocation would confuse an admin reading + // the panel. Short-circuit on !is_active. + let online_cutoff = match chrono::Duration::from_std(ONLINE_WINDOW) { + Ok(d) => Utc::now() - d, + // Cast can only fail on a Duration too large for i64 + // milliseconds; not reachable with our 5 min constant. + // Fall back to "never online" rather than panic — a + // wrong badge is fixable, a request-path panic is not. + Err(_) => DateTime::::MAX_UTC, + }; + let is_online = is_active && s.last_seen_at() > online_cutoff; Self { id: s.id(), user_id: s.user_id(), created_at: s.created_at(), expires_at: s.expires_at(), + last_seen_at: s.last_seen_at(), ip_address: s.ip_address().map(str::to_owned), user_agent: s.user_agent().map(str::to_owned), is_bound: jkt.is_some(), dpop_jkt_prefix, is_revoked, - is_active: !is_revoked && !is_expired, + is_active, + is_online, origin: s.origin(), is_current, } @@ -166,6 +227,7 @@ mod tests { Some(sid.to_string()), None, crate::domain::entities::session::SessionOrigin::Oidc, + Utc::now(), ) } @@ -241,6 +303,121 @@ mod tests { assert!(dto.is_active); } + #[test] + fn dto_exposes_last_seen_at() { + // Regression: the admin table renders "last seen X ago" + // straight off this field, and clients that build + // dashboards off the session API rely on it too. Guards + // against a struct field being removed / renamed silently. + let s = base(false, None); + let expected = s.last_seen_at(); + let dto = SessionSummaryDto::from(s); + assert_eq!(dto.last_seen_at, expected); + let json = serde_json::to_string(&dto).unwrap(); + assert!( + json.contains("\"last_seen_at\""), + "wire shape must include `last_seen_at`: {json}" + ); + } + + /// A freshly-minted, unbound, unrevoked session ships with + /// `last_seen_at = Utc::now()` from `Session::new`, so it + /// MUST render as online. This is the green-dot happy path + /// the admin panel keys off — regression here means the + /// dashboard misses every currently-active session. + #[test] + fn dto_is_online_when_last_seen_is_fresh() { + let dto = SessionSummaryDto::from(base(false, None)); + assert!(dto.is_online, "fresh session must be online: {dto:?}"); + assert!(dto.is_active); + } + + /// A session whose `last_seen_at` is older than the + /// [`ONLINE_WINDOW`] MUST render as offline even when the + /// row is otherwise Active — that's the whole point of the + /// presence vs lifecycle split. Constructed via `from_raw` + /// so we can stamp a stale timestamp deterministically. + #[test] + fn dto_is_not_online_when_last_seen_is_stale() { + let stale = Utc::now() - chrono::Duration::hours(1); + let s = Session::from_raw( + Uuid::new_v4(), + Uuid::new_v4(), + "rt".to_string(), + Utc::now() + Duration::days(30), + None, + None, + stale, + false, + Uuid::new_v4(), + None, + None, + None, + SessionOrigin::Password, + stale, + ); + let dto = SessionSummaryDto::from(s); + assert!(!dto.is_online, "1h-idle session must not be online"); + assert!(dto.is_active, "stale-but-alive session stays active"); + } + + /// Anti-confusion guard: a revoked row whose `last_seen_at` + /// happens to be fresh (the last request that landed just + /// before revocation) must NOT surface as "Online" — an admin + /// reading the panel post-revocation expects the green dot + /// gone. `is_online` short-circuits on `!is_active`. + #[test] + fn dto_is_not_online_when_revoked_even_if_fresh() { + let dto = SessionSummaryDto::from(base(true, None)); + assert!(dto.is_revoked); + assert!(!dto.is_active); + assert!( + !dto.is_online, + "revoked-but-fresh row must never render as online", + ); + } + + /// Same anti-confusion guard for expiry: a session that's + /// past `expires_at` but whose last request landed in the + /// last 5 min must not surface as online. + #[test] + fn dto_is_not_online_when_expired_even_if_fresh() { + let past = Utc::now() - Duration::days(1); + let s = Session::from_raw( + Uuid::new_v4(), + Uuid::new_v4(), + "rt".to_string(), + past, // expires_at in the past + None, + None, + past, + false, + Uuid::new_v4(), + None, + None, + None, + SessionOrigin::Password, + Utc::now(), // last_seen_at fresh + ); + let dto = SessionSummaryDto::from(s); + assert!(!dto.is_active, "expired session is not active"); + assert!( + !dto.is_online, + "expired-but-fresh row must never render as online", + ); + } + + #[test] + fn fresh_session_has_last_seen_equal_to_created_at() { + // The DB default is `NOW()` and `Session::new` mirrors + // that with `Utc::now()` for BOTH columns — so a + // freshly-minted session immediately counts as "recently + // active" for the liveness gauges rather than showing up + // as long-idle for the first flush interval. + let s = base(false, None); + assert_eq!(s.created_at(), s.last_seen_at()); + } + #[test] fn from_raw_expired_session_is_not_active() { let past = Utc::now() - Duration::days(1); @@ -258,6 +435,7 @@ mod tests { None, None, crate::domain::entities::session::SessionOrigin::Unknown, + past, ); let dto = SessionSummaryDto::from(s); assert!(!dto.is_active); diff --git a/src/application/dtos/settings_dto.rs b/src/application/dtos/settings_dto.rs index 0d2cc9da..7917d519 100644 --- a/src/application/dtos/settings_dto.rs +++ b/src/application/dtos/settings_dto.rs @@ -108,14 +108,15 @@ pub struct AdminResetPasswordDto { pub new_password: String, } -/// Query parameters for listing users +/// Query parameters for listing users. `/api/admin/users` used to +/// bifurcate on `?summary=` (flat `PublicUserDto` vs nested +/// `FullUserDto`); that split was retired — the endpoint now always +/// returns `FullUserDto`. Unknown query params are ignored, so +/// existing callers still passing `?summary=true` keep working. #[derive(Debug, Serialize, Deserialize)] pub struct ListUsersQueryDto { pub limit: Option, pub offset: Option, - /// Return only the fields rendered by the paginated management table. - /// Defaults to `false` so existing API clients keep the full user shape. - pub summary: Option, } /// Query parameters for the admin sessions listing. @@ -163,10 +164,39 @@ pub struct DashboardStatsDto { pub auth_enabled: bool, pub oidc_configured: bool, pub quotas_enabled: bool, - // User stats + // ── User accounts (static breakdown of auth.users) ── + // All four are counts of the SAME table under different + // predicates. `active`, `admin`, `external` are all subsets of + // `total`. `external` is disjoint from `admin` by DB constraint + // (`users_external_not_admin`). The dashboard renders these as + // one grouped section separate from the live-activity section + // below, so admins don't confuse "as-of-now row count" with + // "who's here right now". pub total_users: i64, pub active_users: i64, pub admin_users: i64, + /// Grant-only accounts (magic-link / OIDC-only / OCM recipients). + /// Filtered out of `total_users` / `active_users` since those + /// columns count operational seats (see the SELECT comment). Here + /// as its own metric because operators of external-heavy + /// deployments (public shares, invited-collab shops) need to see + /// the invited population at a glance. + pub external_users: i64, + // ── Live activity (projection over auth.sessions) ── + // Both fields change minute-to-minute, unlike the user counts + // above which only move on register/deactivate/role-toggle. + // Same 5-min window as the Prometheus gauges + // (`oxicloud_sessions_online[_users]` in + // `session_liveness_gauges.rs`), computed via the shared + // `ONLINE_WINDOW` constant so per-user badges + aggregate + // counts + this dashboard number stay consistent by construction. + /// Distinct users behind non-revoked sessions active in the last + /// 5 min. Answers "how many humans are here right now?". + pub online_users: i64, + /// Non-revoked sessions active in the last 5 min. Answers "how + /// many concurrent connections must I serve?". Ratio + /// `online_sessions / online_users` is the multi-device factor. + pub online_sessions: i64, // ── Per-drive-kind quota accounting ── // One row per drive kind (personal, shared). Pre-dedup, logical // file sizes summed from `drives.used_bytes` (personal rolls up diff --git a/src/application/dtos/user_dto.rs b/src/application/dtos/user_dto.rs index 9147fafa..6822535d 100644 --- a/src/application/dtos/user_dto.rs +++ b/src/application/dtos/user_dto.rs @@ -1,5 +1,4 @@ use crate::domain::entities::user::User; -use crate::domain::repositories::user_repository::UserListEntry; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use smol_str::SmolStr; @@ -7,287 +6,283 @@ use std::sync::Arc; use utoipa::ToSchema; use uuid::Uuid; +// ──────────────────────────────────────────────────────────────────────── +// Three-layer user DTO family — see docs/plan/userdto-refactor.md. +// +// `PublicUserDto` — public identity. Every authenticated caller may see it. +// Returned by /api/users/{id}, share responses, group +// members, magic-link invitees, recipient enrichment. +// `FullUserDto` — `{ user: PublicUserDto, ...admin+self extras }`. +// Returned as `Vec` by /api/admin/users; +// embedded in `SelfUserDto`. Closest DTO to the +// `auth.users` row. +// `SelfUserDto` — `{ full: FullUserDto, ...self-only extras }`. Returned +// by /api/auth/me and by every AuthResponseDto path. +// +// Adding a field? Decide by audience: +// * Any authenticated caller may see it about another user → `PublicUserDto`. +// * Only admin (about another user) AND self (about self) → `FullUserDto`. +// * Only self about themselves → `SelfUserDto`. +// ──────────────────────────────────────────────────────────────────────── + +/// Public identity — what any authenticated caller may see about ANOTHER +/// user. Returned by `/api/users/{id}` and everywhere a user is +/// referenced by another surface (share responses, group members, +/// magic-link invitees, recipient enrichment). +/// +/// This is the audience-narrowest DTO: adding a field here means every +/// authenticated caller can see it about every visible user. Fields that +/// are meaningful only to the subject themselves (preferences, session +/// state) or only to an admin (auth adoption signals) belong on +/// [`SelfUserDto`] or [`FullUserDto`] respectively. #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct UserDto { +pub struct PublicUserDto { pub id: String, - /// Optional handle. `None` for users who have not claimed one - /// (externals, fresh email-only signups). Frontend display callers - /// should walk `username → given/family → email` as their fallback - /// chain. Omitted from JSON when None (consistent with the existing - /// given_name / family_name fields). #[serde(skip_serializing_if = "Option::is_none")] pub username: Option, pub email: String, + /// Role string ("admin" | "user"). Kept public because the sharee / + /// group-member vignette renders an admin badge. pub role: String, - pub storage_quota_bytes: i64, - pub storage_used_bytes: i64, - pub created_at: DateTime, - pub updated_at: DateTime, - pub last_login_at: Option>, - pub active: bool, - /// Which trust chain minted this user's federation identity — - /// `"oidc" | "ocm" | "magic_link"` — or `None` for pure local - /// users. Load-bearing for "is this user OIDC?"-shape predicates: - /// use `federation_kind == "oidc"` rather than string-scraping - /// `federation_issuer`. Serialized only when populated. - /// - /// Mirrors `auth.users.federation_kind` verbatim — same name at - /// DB, entity, and wire layers so there's no translation to reason - /// about. See docs/plan/ocm.md § Identity & auth model. - #[serde(skip_serializing_if = "Option::is_none")] - pub federation_kind: Option, - /// The authority that mints this user's `federation_subject` — - /// issuer URL for OIDC (id_token `iss` claim), peer domain for - /// OCM, `null` for local users (password / OPAQUE only). - /// - /// Renamed from `auth_provider` (which was a `String` with the - /// sentinel `"local"` for non-federated users, and a human-readable - /// label like `"MockSSO"` before Phase B). This shape mirrors the - /// `auth.users.federation_issuer` column directly: nullable when - /// there's no federation involved. FE predicates for "is this user - /// federated?" should read `federation_kind`, not - /// string-compare this value. - /// - /// When populated, FE code that wants a friendly display label - /// looks this value up against `OidcProviderInfoDto.issuer → - /// provider_name` to render the deployment's configured display - /// name; falls back to the raw issuer for foreign IdPs / legacy - /// rows still holding a pre-Phase-B label. - #[serde(skip_serializing_if = "Option::is_none")] - pub federation_issuer: Option, + /// Avatar payload (base64 data-URI up to 512 KiB). Public so a share + /// picker can render the recipient's face directly. Will move to a + /// dedicated avatar endpoint in a future refactor — this shape is + /// transitional. pub image: Option, - pub can_edit_image: bool, /// `true` for grant-only external recipients (magic-link, OIDC-only, - /// future OCM federated). External users have no home folder and - /// can't own storage; their quota is always 0. Internal users - /// default to `false`. + /// future OCM federated). Renders the "external" badge on the vignette. pub is_external: bool, - /// Optional first/given name. Populated from the OIDC `given_name` - /// claim at JIT provisioning, or via a profile-edit endpoint. - /// `None` until explicitly set — `skip_serializing_if = "Option::is_none"` - /// keeps the wire format compact for the common case. + /// Optional first/given name. Social identity. #[serde(skip_serializing_if = "Option::is_none")] pub given_name: Option, - /// Optional last/family name. Same provenance + serde rules as - /// `given_name`. + /// Optional last/family name. Social identity. #[serde(skip_serializing_if = "Option::is_none")] pub family_name: Option, - /// When the user first demonstrated control of their email (PR 23). - /// `None` = unverified (omitted from JSON). Stamped on the first - /// successful magic-link redemption or OIDC JIT with verified - /// claim. Idempotent — the original timestamp is preserved on - /// subsequent verifications. - #[serde(skip_serializing_if = "Option::is_none")] - pub email_verified_at: Option>, - /// User-chosen locale for server-rendered surfaces (emails, - /// future authenticated HTML). `None` = no preference (the server - /// resolves to `OXICLOUD_DEFAULT_LOCALE` when rendering). Round-trips - /// through `/api/auth/me` and `PATCH /api/auth/me/profile`. - #[serde(skip_serializing_if = "Option::is_none")] - pub preferred_locale: Option, - /// Whether the user wants an email when someone shares a resource - /// with them. `true` (default) = receive share-notification mails; - /// `false` = grants are still created but no email is sent. Honored - /// only on the plain-notification path — magic-link first-invitations - /// to brand-new external users always send, otherwise the recipient - /// could never claim the share. Round-trips through `/api/auth/me` - /// and `PATCH /api/auth/me/profile`. - pub notify_on_share: bool, - /// Opaque UI preferences bag. Cross-device store for pure UI - /// toggles (hide dotfiles, view mode, sidebar collapse, …). The - /// server never inspects the contents — this DTO field just echoes - /// what was PATCHed via `PATCH /api/auth/me/profile`. Shape is a - /// JSON object; the frontend defines the keys it cares about (see - /// `frontend/src/lib/stores/preferences.svelte.ts`). Always present - /// on the wire; empty bag is `{}`, never `null`. - pub ui_preferences: serde_json::Value, - /// Mirrors `auth.users.force_password_change_at_next_login`. Set - /// TRUE by the admin password-reset flow (see - /// `AuthApplicationService::admin_reset_password`) and cleared by - /// a successful self-service `POST /api/auth/change-password`. - /// - /// Populated only by the `/api/auth/me` handler and the login - /// response minter (via a distinct code path). `From` — used - /// by admin listings, share-recipient responses, group-member DTOs, - /// etc. — leaves it at `false`. The flag is a per-session-account - /// concern (does *this* user need to change their password before - /// they can proceed?), not a general user attribute worth - /// surfacing on every list row. - /// - /// The load-bearing consumer is the SPA's session store: on - /// startup and after every refresh, `/me` returns the current - /// flag value and the SPA's nav-guard blocks navigation to - /// anything but the change-password surface until it flips - /// back to false. Backend enforcement is separate (see the - /// `require_no_password_change_pending` middleware) — this DTO - /// field is what the SPA reads to render the mandatory-mode UI. + /// Presence signal — TRUE when the server observed a request on any + /// of this user's non-revoked sessions within the last + /// [`ONLINE_WINDOW`](crate::application::dtos::session_dto::ONLINE_WINDOW) + /// (5 min). Sourced from an EXISTS subquery when the DTO is built + /// from a list-projection path; single-user endpoints that don't + /// enrich presence ship `false`. #[serde(default)] - pub force_password_change: bool, - /// TRUE when the account has a local Argon2id `password_hash` on - /// file. Distinct from `federation_kind`: an OIDC-linked account - /// (`federation_kind == "oidc"`) can ALSO carry a local password if - /// it was set at signup or later — a hybrid posture. The SPA - /// gates the profile page's change-password card on this flag, - /// so hybrid users can rotate their local password even though - /// they normally sign in via SSO. - /// - /// Populated only by the `/api/auth/me` handler. `From` in - /// this file leaves it `false` — other UserDto emitters (admin - /// listings, share-recipient responses, group members) do not - /// need to surface per-user credential state. - #[serde(default)] - pub has_password: bool, - /// TRUE when the caller's current session carries a DPoP JWK - /// thumbprint (`session.dpop_jkt IS NOT NULL`). Sourced from the - /// caller's JWT `cnf.jkt` claim — `is_some()` means the session - /// was bound at token-mint time. - /// - /// Populated only by the `/api/auth/me` handler; other UserDto - /// emitters leave it `false`. The SPA reads this on `session.load()` - /// to skip a redundant `POST /api/auth/dpop/bind` call when the - /// session is already bound (which would 409 and log noisily under - /// the audit stream — see the `already_bound` reject). Only the - /// OIDC / magic-link redirect flows land here as `false` on first - /// visit; password login binds at session-mint time so the very - /// first `/me` after login already reports `true`. - #[serde(default)] - pub is_dpop_bound: bool, + pub is_online: bool, } -/// Compact row returned by the paginated admin user table. +/// Full user record — public identity + all fields BOTH an admin +/// (viewing another user) AND the subject themselves may see. Returned +/// as `Vec` by `/api/admin/users`; embedded in +/// [`SelfUserDto`] for `/api/auth/me`. /// -/// Account-detail fields deliberately do not appear here. In particular, -/// omitting `image` and `ui_preferences` prevents a 100-row page from turning -/// into tens of MiB when users have uploaded avatars. `GET /api/admin/users/:id` -/// remains the full-detail endpoint. +/// This is the DTO closest to the underlying `auth.users` row. Adding a +/// field here means an admin looking at any user can see it, and the +/// subject themselves can see it in their `/me` response — but the field +/// stays off the public [`PublicUserDto`] surface. #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -pub struct AdminUserSummaryDto { - pub id: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub username: Option, - pub email: String, - pub role: String, - pub storage_quota_bytes: i64, - pub storage_used_bytes: i64, - pub last_login_at: Option>, - pub active: bool, - /// See `UserDto::federation_kind` — same semantics, same wire spelling. +pub struct FullUserDto { + /// Public identity — same set every authenticated caller sees. + pub user: PublicUserDto, + /// Which trust chain minted this user's federation identity — + /// `"oidc" | "ocm" | "magic_link"` — or `None` for pure local users. + /// Kept off `PublicUserDto` because a peer's federation kind is a + /// soft org-affiliation leak; only self + admin need it. #[serde(skip_serializing_if = "Option::is_none")] pub federation_kind: Option, - /// See `UserDto::federation_issuer` — same semantics, same wire spelling. + /// The authority that minted this user's `federation_subject` — + /// issuer URL for OIDC (id_token `iss` claim), peer domain for OCM, + /// `None` for local users. Same rationale as `federation_kind`. #[serde(skip_serializing_if = "Option::is_none")] pub federation_issuer: Option, - pub is_external: bool, - /// TRUE when the user has a server-verifiable password on file - /// (`password_hash IS NOT NULL`). The admin table uses this - /// alongside `federation_issuer` and `opaque_registered` to render - /// the user's full capability set: a `password` chip lights up - /// here, an OIDC provider name renders the SSO badge, an - /// envelope-on-file flips the OPAQUE chip. A user with none of - /// the three is passwordless (magic-link only — the SPA renders - /// a distinct `passwordless` chip in that case). Admin-only - /// exposure — see the DTO doc for why this isn't on `UserDto`. - #[serde(default)] + /// Subject's own locale preference. Only THEY or an admin managing + /// them needs this — other callers use their own locale. + #[serde(skip_serializing_if = "Option::is_none")] + pub preferred_locale: Option, + /// When the user first demonstrated control of their email. Trust + /// signal — meaningful to admin (auditing verification status) and + /// to self (own record), but not to a share picker rendering a + /// vignette. + #[serde(skip_serializing_if = "Option::is_none")] + pub email_verified_at: Option>, + /// Row bookkeeping. + pub created_at: DateTime, + pub updated_at: DateTime, + /// Activity signal — private to the subject; admin sees it too. + pub last_login_at: Option>, + /// Account-active flag — a deactivated user couldn't reach `/me` + /// anyway, but admin needs to see it. + pub active: bool, + /// Storage quotas — personal financials. Admin manages others'; + /// self sees own. + pub storage_quota_bytes: i64, + pub storage_used_bytes: i64, + /// TRUE when the account has a server-verifiable password + /// (`password_hash IS NOT NULL`). Kept off `PublicUserDto` because + /// per-user auth adoption leaks through directory endpoints. pub has_password: bool, - /// Mirrors `UserListEntry::opaque_registered` — TRUE when the user - /// has an OPAQUE envelope on file. Surfaced on the admin table so - /// operators can see per-user rollout progress during the - /// migration window. **Admin-only exposure**: this field is NOT - /// on `UserDto` — putting it there would leak adoption status - /// through every user-directory-adjacent endpoint (share targets, - /// group members, invite listings). `#[serde(default)]` keeps - /// older SPA builds tolerant of the added field. - #[serde(default)] + /// TRUE when the user has an OPAQUE envelope on file. pub opaque_registered: bool, - /// Mirrors `UserListEntry::opaque_migrated` — TRUE when the user - /// has completed at least one successful OPAQUE login. Distinct - /// from `opaque_registered`: an admin can invalidate the envelope - /// (`clear_registration`) leaving the user registered=false but - /// with a historical migrated=true; the SPA's admin table shows - /// both so this operational nuance is visible. - #[serde(default)] + /// TRUE when the user has completed ≥1 successful OPAQUE login. + /// Distinct from `opaque_registered`: an admin can invalidate the + /// envelope leaving the user registered=false but with historical + /// migrated=true. pub opaque_migrated: bool, } -impl From for AdminUserSummaryDto { - fn from(entry: UserListEntry) -> Self { - Self { - id: entry.id.to_string(), - username: entry.username, - email: entry.email, - role: entry.role.to_string(), - storage_quota_bytes: entry.storage_quota_bytes, - storage_used_bytes: entry.storage_used_bytes, - last_login_at: entry.last_login_at, - active: entry.active, - federation_kind: entry.federation_kind, - federation_issuer: entry.federation_issuer, - is_external: entry.is_external, - has_password: entry.has_password, - opaque_registered: entry.opaque_registered, - opaque_migrated: entry.opaque_migrated, - } - } +/// Self view — everything the caller may see about themselves. +/// Returned by `/api/auth/me` and by every `AuthResponseDto` path +/// (login / refresh / OIDC callback / magic-link redemption). +/// +/// Composed on top of [`FullUserDto`] so `/me` and `/admin/users` share +/// the SAME "full profile" contract for the fields both need — new +/// self+admin-visible fields go on `FullUserDto` and both endpoints get +/// them together. Fields here are pure self-scoped state: preferences, +/// session-scoped flags, and caller-scoped permissions. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct SelfUserDto { + /// Full profile — same shape as one row of `/api/admin/users`. + pub full: FullUserDto, + /// Opaque UI preferences bag — my own UI state. Cross-device store + /// for pure UI toggles (view mode, sidebar collapse, hide dotfiles, + /// …). The server never inspects the contents. Always present on + /// the wire; empty bag is `{}`, never `null`. + pub ui_preferences: serde_json::Value, + /// Whether I want share-notification emails. + pub notify_on_share: bool, + /// Session-scoped: my current session carries a DPoP thumbprint. + /// SPA reads this on `session.load()` to skip a redundant + /// `POST /api/auth/dpop/bind` when the session is already bound. + pub is_dpop_bound: bool, + /// Admin-set temp-password gate — SPA nav guard blocks everything + /// but `/change-password` until this flips back. Cleared by a + /// successful `POST /api/auth/change-password`. + pub force_password_change: bool, + /// Caller-scoped permission: can I edit my own avatar? `false` for + /// OIDC users whose avatar comes from the IdP. Only meaningful when + /// caller == subject; nonsense on any other DTO. + pub can_edit_image: bool, } -impl From for UserDto { - fn from(user: User) -> Self { - // `user` is owned and dropped here, so every owned field is MOVED out - // via `into_parts` rather than cloned through the borrowing accessors — - // the accessor form deep-cloned `image` (a data URI up to 512 KiB) and - // the whole `ui_preferences` JSON tree on every `/api/auth/me` and admin - // user listing (benches/ROUND20.md §A2). The two derived values read the - // entity before the move. +impl PublicUserDto { + /// Construct a `PublicUserDto` from a `User` entity + an explicit + /// `is_online` signal. + /// + /// **Why not `From`?** The `User` entity models a row in + /// `auth.users`; `is_online` is a cross-table lookup on + /// `auth.sessions` (see the EXISTS subquery in + /// `list_users_with_derived_flags` and `get_user_with_derived_flags` + /// on the user repo). A `From` impl couldn't compute it + /// honestly — it would have to ship a `false` default that lies to + /// the FE presence dot on every emitter that didn't remember to + /// override. Making presence a required constructor argument + /// removes that footgun: every callsite has to declare its intent. + /// + /// Two shapes at the callsite: + /// + /// - Presence matters (single-user `/api/users/{id}`, list + /// projections, self-view): pair with + /// `user_storage.get_user_with_derived_flags(id)` and pass + /// `flags.is_online`. + /// - Presence is out of scope (register / update-profile response, + /// post-mutation echo where the FE ignores the field): pass + /// `false` with a short comment explaining why. The receiver's + /// presence read is a no-op — no dot lights up on the stale + /// value. + pub fn new(user: User, is_online: bool) -> Self { let role = format!("{}", user.role()); - let can_edit_image = !user.is_oidc_user(); - // has_password is derivable from the entity — read before the - // move. Cheap (bool from Option::is_some), no extra DB round- - // trip, so From can populate it uniformly rather than - // leaving it false and requiring per-call-site backfill. - let has_password = user.has_password(); let p = user.into_parts(); Self { id: p.id.to_string(), username: p.username, email: p.email, role, - storage_quota_bytes: p.storage_quota_bytes, - storage_used_bytes: p.storage_used_bytes, + image: p.image, + is_external: p.is_external, + given_name: p.given_name, + family_name: p.family_name, + is_online, + } + } +} + +impl FullUserDto { + /// Construct a `FullUserDto` from a `User` entity plus the DB-derived + /// flags the entity doesn't carry (`has_password`, OPAQUE flags, + /// `is_online`). Both are typically produced together by the users + /// list repo projection. + /// + /// Not a `From` impl because it takes two arguments; not a `From + /// <(User, UserDerivedFlags)>` because that reads awkwardly at + /// callsites — `FullUserDto::build(user, flags)` is clearer. + pub fn build( + user: User, + flags: crate::domain::repositories::user_repository::UserDerivedFlags, + ) -> Self { + let role = format!("{}", user.role()); + let p = user.into_parts(); + Self { + user: PublicUserDto { + id: p.id.to_string(), + username: p.username, + email: p.email, + role, + image: p.image, + is_external: p.is_external, + given_name: p.given_name, + family_name: p.family_name, + is_online: flags.is_online, + }, + federation_kind: p.federation_kind.map(|k| k.as_str().to_string()), + federation_issuer: p.federation_issuer, + preferred_locale: p.preferred_locale, + email_verified_at: p.email_verified_at, created_at: p.created_at, updated_at: p.updated_at, last_login_at: p.last_login_at, active: p.active, - // NULL on both fields for local users (no federation wired). - // FE predicates use `!!federation_kind` for "is federated?" — - // no "local" sentinel string; the null tells the whole story. - federation_kind: p.federation_kind.map(|k| k.as_str().to_string()), - federation_issuer: p.federation_issuer, - image: p.image, - can_edit_image, - is_external: p.is_external, - given_name: p.given_name, - family_name: p.family_name, - email_verified_at: p.email_verified_at, - preferred_locale: p.preferred_locale, - notify_on_share: p.notify_on_share, - ui_preferences: p.ui_preferences, - // Defaults to false. The `/me` handler + the login-response - // minter populate this via a distinct code path (a - // repo read that goes through the auth service's cache); - // admin listings and other UserDto consumers deliberately - // leave it false — the flag is per-session-account state, - // not a general user attribute. - force_password_change: false, - has_password, - // Populated only by `/api/auth/me` — the handler overlays - // the caller's session's actual DPoP binding state after - // this `From` runs. Other UserDto emitters leave - // this at `false` (they lack session context). - is_dpop_bound: false, + storage_quota_bytes: p.storage_quota_bytes, + storage_used_bytes: p.storage_used_bytes, + has_password: flags.has_password, + opaque_registered: flags.opaque_registered, + opaque_migrated: flags.opaque_migrated, } } } +impl SelfUserDto { + /// Assemble the `/me` response from a `FullUserDto` plus the two + /// session-scoped booleans that can't be derived from `User` alone: + /// the caller's DPoP-binding state (from the JWT `cnf.jkt` claim) + /// and the admin-set force-password-change flag (from the auth + /// service's cache). + /// + /// The other self-only fields (`ui_preferences`, `notify_on_share`, + /// `can_edit_image`) come from `User` and are read off the entity + /// before it's moved into the FullUserDto; this method takes those + /// as explicit parameters so the caller can decide when to read + /// them (typically at the same point they read the DPoP-binding + /// state). + pub fn build( + full: FullUserDto, + ui_preferences: serde_json::Value, + notify_on_share: bool, + is_dpop_bound: bool, + force_password_change: bool, + can_edit_image: bool, + ) -> Self { + Self { + full, + ui_preferences, + notify_on_share, + is_dpop_bound, + force_password_change, + can_edit_image, + } + } +} + +// ──────────────────────────────────────────────────────────────────────── +// End of three-layer user DTO family. +// ──────────────────────────────────────────────────────────────────────── + #[derive(Debug, Serialize, Deserialize, Clone, ToSchema)] pub struct LoginDto { /// Identifier the user typed. Accepts BOTH a username (no `@`) and @@ -425,7 +420,12 @@ impl UpdateProfileDto { #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct AuthResponseDto { - pub user: UserDto, + /// Full self view — identical shape to `/api/auth/me`. Every + /// login / refresh / OIDC-callback / magic-link redemption ships + /// this so the SPA's post-auth state matches its post-`/me` state + /// (no UI race between `AuthResponseDto` and the first `/me` + /// fetch). See `docs/plan/userdto-refactor.md` § Endpoint mapping. + pub user: SelfUserDto, pub access_token: String, pub refresh_token: String, pub token_type: String, @@ -562,7 +562,7 @@ pub struct OidcProviderInfoDto { /// users JIT-provisioned via this IdP. /// /// Populated so the frontend can resolve display: when - /// `UserDto.federation_issuer` equals this `issuer`, render + /// `PublicUserDto.federation_issuer` equals this `issuer`, render /// `provider_name` as the human-friendly label (avoids showing raw /// issuer URLs like `https://sso.example.com/realms/main` in the /// admin badge / profile view). Falls back to the raw issuer when @@ -606,3 +606,142 @@ pub struct OidcUserInfoDto { pub name: Option, pub groups: Vec, } + +#[cfg(test)] +mod three_layer_quarantine { + use super::*; + use serde_json::Value; + + /// Structural-quarantine guard for `SelfUserDto`. The self-only + /// bag (`ui_preferences`, `notify_on_share`, `is_dpop_bound`, + /// `force_password_change`, `can_edit_image`) MUST live at the + /// top level, NOT nested inside `.full` or `.full.user`. If a + /// future refactor accidentally moves one of them down, the + /// wire shape leaks it through every `PublicUserDto` / + /// `FullUserDto` emitter (share responses, group members, + /// `/api/admin/users`, magic-link invitees) — exactly what the + /// three-layer split exists to prevent. Fails loudly here. + #[test] + fn self_only_fields_stay_at_top_level_of_self_user_dto() { + let self_dto = SelfUserDto { + full: FullUserDto { + user: PublicUserDto { + id: "00000000-0000-0000-0000-000000000001".into(), + username: None, + email: "self@example.invalid".into(), + role: "user".into(), + image: None, + is_external: false, + given_name: None, + family_name: None, + is_online: false, + }, + federation_kind: None, + federation_issuer: None, + preferred_locale: None, + email_verified_at: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + last_login_at: None, + active: true, + storage_quota_bytes: 0, + storage_used_bytes: 0, + has_password: true, + opaque_registered: false, + opaque_migrated: false, + }, + ui_preferences: serde_json::json!({}), + notify_on_share: true, + is_dpop_bound: false, + force_password_change: false, + can_edit_image: true, + }; + let json: Value = serde_json::to_value(&self_dto).expect("SelfUserDto serialises"); + assert!( + json.get("ui_preferences").is_some(), + "top-level ui_preferences" + ); + assert!( + json.get("full") + .expect("full block") + .get("ui_preferences") + .is_none(), + "ui_preferences must NOT appear inside `.full`" + ); + assert!( + json.pointer("/full/user/ui_preferences").is_none(), + "ui_preferences must NOT appear inside `.full.user`" + ); + // Same guard for the other self-only fields. + for k in [ + "notify_on_share", + "is_dpop_bound", + "force_password_change", + "can_edit_image", + ] { + assert!(json.get(k).is_some(), "{k} at top level"); + assert!( + json.pointer(&format!("/full/{k}")).is_none(), + "{k} must NOT nest in .full" + ); + assert!( + json.pointer(&format!("/full/user/{k}")).is_none(), + "{k} must NOT nest in .full.user" + ); + } + } + + /// Structural-quarantine guard for `FullUserDto`. Admin-visible + /// extras (`has_password`, OPAQUE flags, `federation_*`, + /// `last_login_at`, `active`, quotas, `preferred_locale`, + /// `email_verified_at`) MUST live at the top level of + /// `FullUserDto`, NOT inside `.user`. If a future refactor + /// accidentally lifts one of them onto `PublicUserDto` (the + /// embedded `user` field), it leaks through `/api/users/{id}` + /// and every other public directory endpoint. + #[test] + fn admin_only_fields_stay_at_top_level_of_full_user_dto() { + let full = FullUserDto { + user: PublicUserDto { + id: "00000000-0000-0000-0000-000000000002".into(), + username: Some("bob".into()), + email: "bob@example.invalid".into(), + role: "user".into(), + image: None, + is_external: false, + given_name: None, + family_name: None, + is_online: false, + }, + federation_kind: None, + federation_issuer: None, + preferred_locale: None, + email_verified_at: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + last_login_at: None, + active: true, + storage_quota_bytes: 10_737_418_240, + storage_used_bytes: 0, + has_password: true, + opaque_registered: false, + opaque_migrated: false, + }; + let json: Value = serde_json::to_value(&full).expect("FullUserDto serialises"); + for k in [ + "has_password", + "opaque_registered", + "opaque_migrated", + "last_login_at", + "active", + "storage_quota_bytes", + "storage_used_bytes", + ] { + assert!(json.get(k).is_some(), "{k} at top level of FullUserDto"); + assert!( + json.pointer(&format!("/user/{k}")).is_none(), + "{k} must NOT nest in .user" + ); + } + } +} diff --git a/src/application/ports/auth_ports.rs b/src/application/ports/auth_ports.rs index cb8564fa..5a7bd955 100644 --- a/src/application/ports/auth_ports.rs +++ b/src/application/ports/auth_ports.rs @@ -3,7 +3,6 @@ use crate::domain::entities::app_password::AppPassword; use crate::domain::entities::device_code::DeviceCode; use crate::domain::entities::session::Session; use crate::domain::entities::user::User; -use crate::domain::repositories::user_repository::UserListEntry; use std::sync::Arc; use uuid::Uuid; @@ -63,6 +62,14 @@ pub struct TokenClaims { /// The DPoP middleware reads it from the already-validated token /// (no DB round trip) to enforce "bound session → proof required". pub dpop_jkt: Option, + /// Session identifier — the `auth.sessions.id` this access token + /// was minted for. Read by the auth middleware to stamp + /// per-session liveness via [`LastSeenTracker`](crate::infrastructure::services::last_seen_tracker) + /// with no DB round trip. `None` for tokens minted by builds + /// that predate the `sid` claim (backward compat during rollout; + /// harmless — the missing sid just means no stamp fires, and + /// the token still authenticates normally). + pub sid: Option, } /// Port for JWT token operations. @@ -81,6 +88,7 @@ pub trait TokenServicePort: Send + Sync + 'static { fn generate_access_token( &self, user: &User, + session_id: Option, dpop_jkt: Option<&str>, ) -> Result; @@ -114,6 +122,36 @@ pub trait UserStoragePort: Send + Sync + 'static { /// Gets a user by ID async fn get_user_by_id(&self, id: Uuid) -> Result; + /// Fetch the full `User` + [`UserDerivedFlags`] in one query. See + /// [`UserRepository::get_user_with_derived_flags`](crate::domain::repositories::user_repository::UserRepository::get_user_with_derived_flags) + /// for the contract and the rationale for the single-query shape. + async fn get_user_with_derived_flags( + &self, + id: Uuid, + ) -> Result< + ( + User, + crate::domain::repositories::user_repository::UserDerivedFlags, + ), + DomainError, + >; + + /// Paginated admin user listing with derived flags. See + /// [`UserRepository::list_users_with_derived_flags`](crate::domain::repositories::user_repository::UserRepository::list_users_with_derived_flags) + /// for the contract and rationale. + async fn list_users_with_derived_flags( + &self, + limit: i64, + offset: i64, + include_external: bool, + ) -> Result< + Vec<( + User, + crate::domain::repositories::user_repository::UserDerivedFlags, + )>, + DomainError, + >; + /// Batch-loads users by id. Order is unspecified; missing ids are /// silently dropped. Used by group-recipient expansion in /// `RecipientNotificationService` to avoid N+1 lookups when notifying @@ -155,15 +193,6 @@ pub trait UserStoragePort: Send + Sync + 'static { include_external: bool, ) -> Result, DomainError>; - /// Narrow user-list projection for management tables. Keeps heavyweight - /// account-detail fields off the database and JSON hot path. - async fn list_user_summaries( - &self, - limit: i64, - offset: i64, - include_external: bool, - ) -> Result, DomainError>; - /// Searches users by username or email (SQL ILIKE) with a limit. /// See [`list_users`] for the meaning of `include_external`. async fn search_users( diff --git a/src/application/ports/blob_reference_ports.rs b/src/application/ports/blob_reference_ports.rs new file mode 100644 index 00000000..aa949ba3 --- /dev/null +++ b/src/application/ports/blob_reference_ports.rs @@ -0,0 +1,313 @@ +//! `BlobReferenceSource` — the extension point that teaches ref-counting +//! and the consistency jobs about a table holding blob references. +//! +//! Before this port, "who references this hash" was hardcoded SQL in two +//! places (`dedup_gc`'s reap predicate and `blobs_consistency`'s refcount +//! recompute), both naming `storage.files` and `storage.chunk_manifests` +//! directly. Any new blob-owning table therefore risked silent orphaning: +//! `dedup_gc` sees `ref_count = 0`, or a manifest with no `storage.files` +//! row behind it, and reaps live content. +//! +//! See `docs/plan/derived-blobs.md` for the design and the coverage matrix. +//! +//! # Two levels, and why a source may span both +//! +//! [`DedupService::add_reference`] bumps `chunk_manifests.ref_count` first +//! and only falls back to `storage.blobs.ref_count`. So a reference lands +//! on whichever counter its hash names, and the two must be recomputed +//! separately — mixing them double-counts, systematically: +//! +//! * A **Blob** (`chunk_manifests.file_hash`) is "the content of a file". +//! * A **Chunk** (`storage.blobs.hash`) is a physical byte payload. +//! * For a single-chunk Blob the two hashes are **equal**, because both are +//! BLAKE3 over the same bytes. That aliasing is why today's chunk-level +//! recompute carries a `NOT EXISTS` clause, and why every fragment here +//! must be level-correct rather than merely plausible. +//! +//! A source is not confined to one level: [`RefLevel::Chunk`] and +//! [`RefLevel::Manifest`] fragments are requested independently, and +//! `storage.files` legitimately contributes to both — a manifest-less +//! legacy row references a chunk, a CDC row references a Blob. +//! +//! # Why SQL fragments rather than a per-hash count +//! +//! `blobs_consistency` recomputes refcounts with **one query per page**, +//! the expected count inlined as correlated subqueries. Asking each source +//! for a count per hash would turn that into `sources × rows` round-trips — +//! a catastrophic regression on a table with millions of rows. So sources +//! contribute a *fragment* that the registry sums into the existing page +//! query, and [`BlobReferenceSource::count_references`] exists only for the +//! on-demand path (`dedup_gc` checking a single reap candidate, where the +//! candidate set is already filtered to `ref_count = 0`). + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::domain::errors::DomainError; + +/// Which counter a source's references land on. +/// +/// Not a property of the source — see the module docs; the same source may +/// contribute at both levels. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RefLevel { + /// References a physical chunk. Feeds `storage.blobs.ref_count`. + Chunk, + /// References a Blob via its manifest. Feeds + /// `chunk_manifests.ref_count`. + Manifest, +} + +impl RefLevel { + /// Both levels, for callers that sweep each in turn. + pub const ALL: [RefLevel; 2] = [RefLevel::Chunk, RefLevel::Manifest]; + + /// Stable name for logs and consistency-finding fields. + pub fn as_str(self) -> &'static str { + match self { + RefLevel::Chunk => "chunk", + RefLevel::Manifest => "manifest", + } + } +} + +/// One table that holds references to blob hashes. +/// +/// Implementors are registered on [`BlobReferenceRegistry`] during DI. +/// Adding a blob-owning table **without** registering it is the failure +/// this port exists to prevent. +#[async_trait] +pub trait BlobReferenceSource: Send + Sync { + /// Short stable identifier for logs and consistency-finding `source` + /// fields — `"files"`, `"chunks"`, `"content_derived"`, … + /// + /// Stable across releases: log aggregators key off it. + fn source_name(&self) -> &'static str; + + /// A correlated-subquery fragment counting this source's references + /// **at `level`** to `outer_hash_expr`, or `None` when this source + /// holds no references at that level. + /// + /// `outer_hash_expr` is the SQL expression naming the hash of the row + /// being recomputed — `"b.hash"` when sweeping `storage.blobs`, + /// `"m.file_hash"` when sweeping `storage.chunk_manifests`. The + /// fragment must be a parenthesised scalar subquery so the registry can + /// join fragments with `+`. + /// + /// **Identifiers only.** `outer_hash_expr` is supplied by the sweep, never + /// by a request; no fragment may interpolate caller input. + fn ref_count_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option; + + /// Existence form of [`Self::ref_count_sql`] — a boolean fragment, true + /// when this source holds at least one reference at `level`. + /// + /// Defaults to `() > 0`. Override when the source can express a + /// short-circuiting `EXISTS`, which the planner can stop at the first + /// matching row: `dedup_gc`'s reap predicate runs this per candidate + /// manifest, and a heavily-deduplicated blob has many referrers, so + /// counting all of them where existence would do is a real regression. + fn ref_exists_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + self.ref_count_sql(level, outer_hash_expr) + .map(|fragment| format!("{fragment} > 0")) + } + + /// Count of references this source holds on `blob_hash`, across both + /// levels. + /// + /// **On-demand path only** — `dedup_gc` checking a single reap + /// candidate. The consistency sweeps must use [`Self::ref_count_sql`]; + /// calling this per row would turn one query per page into + /// `sources × rows` round-trips. + async fn count_references(&self, blob_hash: &str) -> Result; + + /// Iterate the hashes this source references, paged by the + /// implementation's natural cursor (typically a primary key). + /// + /// Used by `backend_consistency` to walk the backend against the union + /// of all sources. Returns the page plus the cursor to resume from, + /// `None` when exhausted. + async fn list_referenced_blobs( + &self, + cursor: Option>, + limit: usize, + ) -> Result<(Vec, Option>), DomainError>; + + /// Notification that `dedup_gc` reaped this blob. + /// + /// Sources maintaining a denormalised refcount can clean up here. Most + /// leave the default noop — the mapping row is normally deleted by the + /// owning service's `on_blob_deleted` hook instead. + fn on_blob_reaped(&self, _blob_hash: &str) {} +} + +/// The set of registered [`BlobReferenceSource`]s. +/// +/// Assembled once during DI and shared (`Arc`) by `dedup_gc` and the +/// consistency jobs, so all three agree on what "referenced" means. +#[derive(Default)] +pub struct BlobReferenceRegistry { + sources: Vec>, +} + +impl BlobReferenceRegistry { + pub fn new() -> Self { + Self::default() + } + + /// Register a source. Order is irrelevant — fragments are summed and + /// counts added. + pub fn register(&mut self, source: Arc) { + self.sources.push(source); + } + + pub fn sources(&self) -> &[Arc] { + &self.sources + } + + /// The summed SQL expression counting every source's references at + /// `level` to `outer_hash_expr`. + /// + /// Returns `"0"` when no source contributes at this level, which keeps + /// the caller's query valid without a special case. + pub fn ref_count_expr(&self, level: RefLevel, outer_hash_expr: &str) -> String { + let fragments: Vec = self + .sources + .iter() + .filter_map(|s| s.ref_count_sql(level, outer_hash_expr)) + .collect(); + + if fragments.is_empty() { + "0".to_string() + } else { + fragments.join("\n + ") + } + } + + /// Predicate selecting rows that **no** registered source references at + /// `level` — i.e. reap candidates. + /// + /// Returns `None` when no source contributes at this level, and callers + /// **must** treat that as "refuse to act" rather than substituting a + /// default. The natural default would be the sum-equals-zero form, which + /// on an empty registry reduces to `0 = 0` — vacuously true for every + /// row, i.e. "delete everything". Returning `None` makes that + /// unrepresentable at the call site instead of merely discouraged. + pub fn no_reference_predicate(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + let fragments: Vec = self + .sources + .iter() + .filter_map(|s| s.ref_exists_sql(level, outer_hash_expr)) + .collect(); + + if fragments.is_empty() { + return None; + } + Some(format!("NOT ({})", fragments.join("\n OR "))) + } + + /// Total references held on `hash` across every source. + /// + /// On-demand path only — see [`BlobReferenceSource::count_references`]. + pub async fn total_references(&self, hash: &str) -> Result { + let mut total = 0u64; + for source in &self.sources { + total = total.saturating_add(source.count_references(hash).await?); + } + Ok(total) + } + + /// Fan out a reap notification to every source. + pub fn notify_reaped(&self, blob_hash: &str) { + for source in &self.sources { + source.on_blob_reaped(blob_hash); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Stub { + name: &'static str, + chunk: Option<&'static str>, + manifest: Option<&'static str>, + count: u64, + } + + #[async_trait] + impl BlobReferenceSource for Stub { + fn source_name(&self) -> &'static str { + self.name + } + + fn ref_count_sql(&self, level: RefLevel, outer: &str) -> Option { + let tmpl = match level { + RefLevel::Chunk => self.chunk?, + RefLevel::Manifest => self.manifest?, + }; + Some(tmpl.replace("{outer}", outer)) + } + + async fn count_references(&self, _blob_hash: &str) -> Result { + Ok(self.count) + } + + async fn list_referenced_blobs( + &self, + _cursor: Option>, + _limit: usize, + ) -> Result<(Vec, Option>), DomainError> { + Ok((Vec::new(), None)) + } + } + + fn registry() -> BlobReferenceRegistry { + let mut r = BlobReferenceRegistry::new(); + r.register(Arc::new(Stub { + name: "a", + chunk: Some("(SELECT 1 WHERE {outer} = 'x')"), + manifest: None, + count: 2, + })); + r.register(Arc::new(Stub { + name: "b", + chunk: Some("(SELECT 2 WHERE {outer} = 'y')"), + manifest: Some("(SELECT 3 WHERE {outer} = 'z')"), + count: 5, + })); + r + } + + #[test] + fn chunk_level_sums_every_contributing_source() { + let expr = registry().ref_count_expr(RefLevel::Chunk, "b.hash"); + assert!(expr.contains("b.hash = 'x'"), "{expr}"); + assert!(expr.contains("b.hash = 'y'"), "{expr}"); + assert!(expr.contains('+'), "fragments must be summed: {expr}"); + } + + /// A source returning `None` for a level must contribute nothing there — + /// this is what keeps manifest-only tables out of the chunk recompute, + /// where they would double-count against the single-chunk hash alias. + #[test] + fn manifest_level_skips_non_contributing_sources() { + let expr = registry().ref_count_expr(RefLevel::Manifest, "m.file_hash"); + assert!(expr.contains("m.file_hash = 'z'"), "{expr}"); + assert!(!expr.contains('+'), "only one source contributes: {expr}"); + } + + /// An empty level must still yield a valid scalar expression, so callers + /// need no special case before a registry is fully populated. + #[test] + fn empty_level_yields_zero_literal() { + let r = BlobReferenceRegistry::new(); + assert_eq!(r.ref_count_expr(RefLevel::Chunk, "b.hash"), "0"); + } + + #[tokio::test] + async fn total_references_adds_across_sources() { + assert_eq!(registry().total_references("deadbeef").await.unwrap(), 7); + } +} diff --git a/src/application/ports/blob_storage_ports.rs b/src/application/ports/blob_storage_ports.rs index 262a1321..ae8732d3 100644 --- a/src/application/ports/blob_storage_ports.rs +++ b/src/application/ports/blob_storage_ports.rs @@ -244,11 +244,28 @@ pub trait BlobStorageBackend: Send + Sync + 'static { /// /// * `cursor` — opaque continuation token from a prior call, or /// `None` to start from the beginning. Format is per-backend - /// (local = last path visited; S3 = continuation token; Azure - /// = list marker); callers treat it as opaque. + /// — **the last blob hash returned by the previous page**. + /// Enumeration resumes strictly AFTER that hash. + /// + /// This is deliberately NOT an opaque backend token. Callers may + /// synthesise a cursor from any hash they hold, which is what lets a + /// consistency sweep merge-join this stream against a + /// `storage.blobs` walk and resume both sides from one checkpoint. + /// An opaque token would force the backend side to re-enumerate from + /// the beginning on every resume. /// * `limit` — soft cap on batch size; backends may return /// fewer (e.g. end of a shard directory). /// + /// **Entries MUST be returned in ascending hash order**, and pages must + /// be contiguous in that order. Every shipped backend already satisfies + /// this — local sorts within each shard and walks shards `00`..`ff` + /// (the shard IS the hash prefix, so that is globally sorted); S3 and + /// Azure list lexicographically by key, and `blobs//` sorts + /// identically to ``. It is stated here because the merge-join in + /// `backend_consistency` depends on it: an unordered backend would + /// silently emit bogus `blob_missing_from_backend` findings at + /// `data_loss` severity. + /// /// Returns `(entries, next_cursor)`. `next_cursor = None` means /// enumeration is complete. Each `BackendBlobEntry` carries the /// hash + optional mtime for grace-window filtering. diff --git a/src/application/ports/dedup_ports.rs b/src/application/ports/dedup_ports.rs index 36deed77..21910c8a 100644 --- a/src/application/ports/dedup_ports.rs +++ b/src/application/ports/dedup_ports.rs @@ -24,6 +24,39 @@ pub struct BlobMetadataDto { pub content_type: Option, } +/// A stored server-derived artifact: which blob holds it, and what it is. +/// +/// `content_type` is carried so the read path can set the response header +/// without byte-sniffing the payload, which is what it does today. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DerivedBlobRef { + pub blob_hash: String, + pub content_type: String, +} + +/// What the derived tier knows about one `(source_hash, kind, variant)`. +/// +/// Three answers, not two. `Option` could only say +/// "have it" or "don't", which collapses the two cases that matter most +/// to a caller deciding whether to spend a decode: +/// +/// * [`Missing`](Self::Missing) — never attempted. Derive it. +/// * [`NotDerivable`](Self::NotDerivable) — attempted, and the attempt +/// is known to be a waste for this content: the transcode came out +/// larger than the original, the source cannot be decoded, the source +/// is over the decode ceiling. Serve the original and do not retry. +/// * [`Found`](Self::Found) — here are the bytes. +/// +/// Only failures that are deterministic in the CONTENT may be recorded +/// as `NotDerivable`. A timeout or an I/O error is a property of the +/// moment; persisting one would mark a good image underivable forever. +#[derive(Debug, Clone)] +pub enum DerivedLookup { + Missing, + NotDerivable, + Found(DerivedBlobRef), +} + /// Result of a deduplication store operation. #[derive(Debug, Clone)] pub enum DedupResultDto { @@ -83,6 +116,17 @@ pub trait DedupPort: Send + Sync + 'static { /// Check if a blob with the given hash exists. async fn blob_exists(&self, hash: &str) -> bool; + /// Look up a server-derived artifact by the content it was derived from. + /// + /// The read counterpart of `store_derived_blob`. Returns `None` when no + /// such variant has been derived yet — the caller then renders it. + async fn find_derived_blob( + &self, + source_hash: &str, + kind: &str, + variant: &str, + ) -> Option; + /// Get metadata for a blob. async fn get_blob_metadata(&self, hash: &str) -> Option; diff --git a/src/application/ports/mod.rs b/src/application/ports/mod.rs index 3ee7c4dc..53c8e8db 100644 --- a/src/application/ports/mod.rs +++ b/src/application/ports/mod.rs @@ -1,6 +1,7 @@ pub mod auth_ports; pub mod authorization_ports; pub mod blob_lifecycle; +pub mod blob_reference_ports; pub mod blob_storage_ports; pub mod cache_ports; pub mod calendar_ports; diff --git a/src/application/ports/thumbnail_ports.rs b/src/application/ports/thumbnail_ports.rs index 2bd20c9b..1fa7cff7 100644 --- a/src/application/ports/thumbnail_ports.rs +++ b/src/application/ports/thumbnail_ports.rs @@ -77,6 +77,12 @@ pub enum ThumbnailFormat { } impl ThumbnailFormat { + /// Every format, for callers that must handle all of them — notably + /// `thumb_derived_import`, which claims one sidecar extension per format + /// and would silently strand a codec if this list and the write path + /// drifted apart. + pub const ALL: [ThumbnailFormat; 2] = [ThumbnailFormat::Webp, ThumbnailFormat::Jpeg]; + /// Stable name, byte-identical to the derived `Debug` output (see /// [`ThumbnailSize::as_str`] — same ETag-stability contract). pub fn as_str(self) -> &'static str { @@ -94,6 +100,18 @@ impl ThumbnailFormat { } } + /// Media type, for `content_derived_blobs.content_type` and for any + /// response serving these bytes. + /// + /// Beside `ext` deliberately: the two must agree, and an extension + /// without a matching media type is how a WebP ends up labelled JPEG. + pub fn mime(self) -> &'static str { + match self { + ThumbnailFormat::Webp => "image/webp", + ThumbnailFormat::Jpeg => "image/jpeg", + } + } + /// Pick the output format from a request `Accept` header: WebP when the /// client advertises `image/webp`, JPEG otherwise. A plain substring check /// is sufficient — no client sends `image/webp;q=0`, and every WebP-capable diff --git a/src/application/ports/transcode_ports.rs b/src/application/ports/transcode_ports.rs index 1e7ed3b9..f727e8b0 100644 --- a/src/application/ports/transcode_ports.rs +++ b/src/application/ports/transcode_ports.rs @@ -85,9 +85,15 @@ pub trait ImageTranscodePort: Send + Sync + 'static { /// Returns `(content, mime_type, was_transcoded)`. /// If transcoding is not beneficial (output larger than input), returns the /// original content with `was_transcoded = false`. + /// + /// `source_hash` is the BLAKE3 of the original content, which is how the + /// durable derived tier is keyed. `None` restricts the implementation to + /// its local cache — correct for callers with no hash (external mounts), + /// and the behaviour of every caller before that tier existed. async fn get_transcoded( &self, file_id: &str, + source_hash: Option<&str>, original_content: Bytes, original_mime: &str, target_format: OutputFormat, diff --git a/src/application/services/auth_application_service.rs b/src/application/services/auth_application_service.rs index 6bd62e07..618fdd22 100644 --- a/src/application/services/auth_application_service.rs +++ b/src/application/services/auth_application_service.rs @@ -1,6 +1,6 @@ use crate::application::dtos::user_dto::{ - AdminUserSummaryDto, AuthResponseDto, ChangePasswordDto, LoginDto, RefreshTokenDto, - RegisterDto, UpgradeToInternalDto, UserDto, + AuthResponseDto, ChangePasswordDto, FullUserDto, LoginDto, PublicUserDto, RefreshTokenDto, + RegisterDto, SelfUserDto, UpgradeToInternalDto, }; use crate::application::ports::auth_ports::{ OidcIdClaims, OidcServicePort, PasswordHasherPort, SessionStoragePort, TokenServicePort, @@ -319,11 +319,11 @@ pub enum OidcCallbackResult { #[derive(Debug, Clone)] pub enum RegisterResult { /// Boxed to avoid the `large_enum_variant` clippy warning — - /// `UserDto` is ~250 bytes, the other variants are zero-sized, + /// `PublicUserDto` is ~250 bytes, the other variants are zero-sized, /// so a heap-pointer indirection keeps the enum's stack size /// small. `register` is called once per request; the /// allocation cost is negligible. - Created(Box), + Created(Box), UsernameTaken, EmailTaken, } @@ -876,8 +876,9 @@ impl AuthApplicationService { is_external = false, "🛂 user registered", ); - Ok(RegisterResult::Created(Box::new(UserDto::from( + Ok(RegisterResult::Created(Box::new(PublicUserDto::new( created_user, + false, )))) } @@ -894,7 +895,7 @@ impl AuthApplicationService { username: String, email: String, password: String, - ) -> Result { + ) -> Result { // Validate username if username.len() < 3 || username.len() > 254 { return Err(DomainError::new( @@ -981,7 +982,7 @@ impl AuthApplicationService { username, created_user.id() ); - Ok(UserDto::from(created_user)) + Ok(PublicUserDto::new(created_user, false)) } pub async fn login( @@ -1269,21 +1270,17 @@ impl AuthApplicationService { None => None, }; - // Generate tokens using the injected token service. The - // access token carries the `cnf.jkt` binding when present, - // so the DPoP middleware can enforce "bound → proof required" - // straight from the already-validated JWT — no session-row - // lookup on the hot path. - let access_token = self - .token_service - .generate_access_token(&user, validated_jkt.as_deref())?; - - let refresh_token = self.token_service.generate_refresh_token(); - + // Construct the session FIRST so its id is available to the + // token mint below — the `sid` claim lets the auth middleware + // stamp per-session liveness with no DB round trip. Order + // was reversed as part of the `last_seen_at` wiring + // (`docs/plan/sessions.md`). + // // Save session — new login starts a new token family. DPoP // binding is set at INSERT time and immutable thereafter (see // `docs/plan/dpop.md` — a mutable bind would let an attacker // downgrade a bound session by re-binding to their own key). + let refresh_token = self.token_service.generate_refresh_token(); let mut session = Session::new( user.id(), refresh_token.clone(), @@ -1293,6 +1290,18 @@ impl AuthApplicationService { Uuid::new_v4(), origin, ); + + // Generate tokens using the injected token service. The + // access token carries the `cnf.jkt` binding when present, + // so the DPoP middleware can enforce "bound → proof required" + // straight from the already-validated JWT — no session-row + // lookup on the hot path. `sid` correlates the token to the + // session row constructed just above. + let access_token = self.token_service.generate_access_token( + &user, + Some(session.id()), + validated_jkt.as_deref(), + )?; if let Some(jkt) = validated_jkt { // Success-path audit — records the bind so operators can // correlate a session_id in the panel with the exact moment @@ -1312,12 +1321,16 @@ impl AuthApplicationService { session = session.with_dpop_jkt(jkt); } + // Build the SelfUserDto BEFORE `session` moves into + // `create_session` — the builder reads `session.dpop_jkt()`. + let user_id = user.id(); + let user_dto = self.build_self_user_dto(user_id, &session).await?; self.session_storage.create_session(session).await?; // Authentication response - let force_password_change = self.read_force_password_change(user.id()).await; + let force_password_change = self.read_force_password_change(user_id).await; Ok(AuthResponseDto { - user: UserDto::from(user), + user: user_dto, access_token, refresh_token, token_type: "Bearer".to_string(), @@ -1326,6 +1339,69 @@ impl AuthApplicationService { }) } + /// Assemble a `SelfUserDto` for the given user + the session that + /// mints them. Called by every `AuthResponseDto` path + /// (login / refresh / OIDC / magic-link) so the wire shape stays + /// consistent across login flavours and matches what `/api/auth/me` + /// would return. + /// + /// Costs one wide SELECT (`get_user_with_derived_flags`) even when + /// the caller already has a `User` in hand — acceptable because + /// `/login`, `/refresh`, and the OIDC/magic-link callbacks are + /// not hot inner loops. In exchange the composition stays uniform + /// across all four callsites and OPAQUE / `is_online` flags land + /// on the wire without a second lookup at each site. + /// + /// `is_dpop_bound` is derived from the session's own DPoP + /// thumbprint — the session was just constructed, so this reads + /// exactly the binding that will govern subsequent requests. + async fn build_self_user_dto( + &self, + user_id: Uuid, + session: &crate::domain::entities::session::Session, + ) -> Result { + // Session-context flavour — delegates to the shared builder + // with the DPoP-bound flag derived from the session row's + // thumbprint. See [`build_self_user_dto_for_id`] for the + // handler-context flavour. + self.build_self_user_dto_for_id(user_id, session.dpop_jkt().is_some()) + .await + } + + /// Handler-context variant of [`build_self_user_dto`]. Called by + /// every endpoint that returns a `SelfUserDto` from a REST handler + /// (`GET /me`, `PATCH /me/profile`, `POST /upgrade-to-internal`) + /// so the wire shape is byte-for-byte identical across them — + /// avoids a "quiet lie" where a client PATCHes one shape and + /// reads another on the very next `/me`. + /// + /// `is_dpop_bound` is passed in by the handler because the JWT + /// `cnf.jkt` claim is where handler-scope code learns the caller's + /// binding state (via `auth_user.dpop_jkt.is_some()`). Session- + /// mint paths use [`build_self_user_dto`] and derive the flag from + /// the freshly-created `Session` row instead. + pub async fn build_self_user_dto_for_id( + &self, + user_id: Uuid, + is_dpop_bound: bool, + ) -> Result { + let (user, flags) = + UserStoragePort::get_user_with_derived_flags(&*self.user_storage, user_id).await?; + let can_edit_image = !user.is_oidc_user(); + let ui_preferences = user.ui_preferences().clone(); + let notify_on_share = user.notify_on_share(); + let force_password_change = self.read_force_password_change(user_id).await; + let full = FullUserDto::build(user, flags); + Ok(SelfUserDto::build( + full, + ui_preferences, + notify_on_share, + is_dpop_bound, + force_password_change, + can_edit_image, + )) + } + /// Read `force_password_change_at_next_login` for the given user, /// with fail-open semantics on repo error (returns `false` and /// logs a warn). Every callsite that builds an `AuthResponseDto` @@ -1562,8 +1638,9 @@ impl AuthApplicationService { // thread `dpop_jkt` into a GET body. Session is minted // unbound; the SPA calls `POST /api/auth/dpop/bind` // post-redirect to bind it (see Gate 3). Token accordingly - // ships without `cnf.jkt`. - let access_token = self.token_service.generate_access_token(&user, None)?; + // ships without `cnf.jkt`. Session constructed first so + // its id can feed the token's `sid` claim — see the login + // path above for the rationale. let refresh_token = self.token_service.generate_refresh_token(); let session = Session::new( user.id(), @@ -1574,22 +1651,32 @@ impl AuthApplicationService { Uuid::new_v4(), crate::domain::entities::session::SessionOrigin::MagicLink, ); + let access_token = + self.token_service + .generate_access_token(&user, Some(session.id()), None)?; + // Snapshot fields still needed for logging + DTO before + // `session` and `user` are consumed by the storage call and + // the DTO builder below. + let user_id = user.id(); + let user_display = user.display_for_audit().to_string(); + let is_external = user.is_external(); + let user_dto = self.build_self_user_dto(user_id, &session).await?; self.session_storage.create_session(session).await?; tracing::info!( target: "audit", event = "magic_link.redeemed", - user_id = %user.id(), - username = %user.display_for_audit(), - is_external = user.is_external(), + user_id = %user_id, + username = %user_display, + is_external = is_external, resource_kind = ?mlt.resource_kind(), resource_id = ?mlt.resource_id(), cross_browser_confirmed = cross_browser_confirmed, ); - let force_password_change = self.read_force_password_change(user.id()).await; + let force_password_change = self.read_force_password_change(user_id).await; let auth = AuthResponseDto { - user: UserDto::from(user), + user: user_dto, access_token, refresh_token, token_type: "Bearer".to_string(), @@ -1715,16 +1802,12 @@ impl AuthApplicationService { )); } - // Generate new tokens. Inherit the DPoP binding from the - // parent session so the refreshed access token carries the - // same `cnf.jkt` — otherwise every refresh would silently - // downgrade to unbound and the next request would 401 under - // Gate 9 enforcement (see Gate 7). - let access_token = self - .token_service - .generate_access_token(&user, session.dpop_jkt())?; - let new_refresh_token = self.token_service.generate_refresh_token(); - + // Rotate the session first so the new row's id is available + // to the token mint below — the `sid` claim tracks the + // freshly-inserted row, not the revoked parent. Order + // reversed as part of the `last_seen_at` wiring + // (`docs/plan/sessions.md`). + // // New session inherits the family_id so reuse of any ancestor triggers // full-family revocation. Revoking the old session and inserting the // new one happen in ONE transaction (`rotate_session`) — this path @@ -1738,6 +1821,7 @@ impl AuthApplicationService { // refresh silently downgrade the session to unbound, and every // subsequent request would fail DPoP verification once required // mode enforces per-session binding. + let new_refresh_token = self.token_service.generate_refresh_token(); let mut new_session = Session::new( user.id(), new_refresh_token.clone(), @@ -1770,6 +1854,22 @@ impl AuthApplicationService { new_session = new_session.with_oidc_sid(sid.to_string()); } + // Mint the access token AFTER the new session is fully + // configured — the `sid` claim points at the new row's id, + // and `cnf.jkt` inherits from the parent so DPoP proof + // enforcement (Gate 9) still holds across the rotation. + let access_token = self.token_service.generate_access_token( + &user, + Some(new_session.id()), + session.dpop_jkt(), + )?; + + // Build the SelfUserDto before `new_session` is consumed by + // the rotate call — the builder reads `session.dpop_jkt()` + // to compute `is_dpop_bound`. + let user_id = user.id(); + let user_dto = self.build_self_user_dto(user_id, &new_session).await?; + self.session_storage .rotate_session(session.id(), new_session) .await?; @@ -1779,9 +1879,9 @@ impl AuthApplicationService { // initial login. The SPA's post-refresh flow (silent, on // its own timer) can then route the user to change-password // without waiting for an explicit re-login. - let force_password_change = self.read_force_password_change(user.id()).await; + let force_password_change = self.read_force_password_change(user_id).await; Ok(AuthResponseDto { - user: UserDto::from(user), + user: user_dto, access_token, refresh_token: new_refresh_token, token_type: "Bearer".to_string(), @@ -2006,7 +2106,7 @@ impl AuthApplicationService { &self, caller_id: Uuid, dto: UpgradeToInternalDto, - ) -> Result { + ) -> Result { let mut user = self.user_storage.get_user_by_id(caller_id).await?; // Precondition: caller is currently external. Fast-path 409 so @@ -2107,7 +2207,7 @@ impl AuthApplicationService { lc.dispatch_upgraded_to_internal(&updated).await; } - Ok(UserDto::from(updated)) + Ok(PublicUserDto::new(updated, false)) } /// Admin-driven external → internal promotion. @@ -2136,7 +2236,7 @@ impl AuthApplicationService { &self, admin_id: Uuid, target_id: Uuid, - ) -> Result { + ) -> Result { let mut user = self.user_storage.get_user_by_id(target_id).await?; if !user.is_external() { @@ -2218,7 +2318,7 @@ impl AuthApplicationService { "👮🏻‍♂️ external user promoted to internal by admin", ); - Ok(UserDto::from(updated)) + Ok(PublicUserDto::new(updated, false)) } /// `keep_session_id` — when `Some`, revoke every OTHER session for @@ -2473,9 +2573,9 @@ impl AuthApplicationService { Ok(()) } - pub async fn get_user(&self, user_id: Uuid) -> Result { + pub async fn get_user(&self, user_id: Uuid) -> Result { let user = self.user_storage.get_user_by_id(user_id).await?; - Ok(UserDto::from(user)) + Ok(PublicUserDto::new(user, false)) } /// Cached, image-free lookup of the caller's authorization flags @@ -2624,7 +2724,7 @@ impl AuthApplicationService { caller_id: Uuid, dto: crate::application::dtos::user_dto::UpdateProfileDto, locale_registry: &crate::common::locale::LocaleRegistry, - ) -> Result { + ) -> Result { let mut user = self.user_storage.get_user_by_id(caller_id).await?; // For OIDC-managed users, refuse the patch ONLY when it touches @@ -2800,7 +2900,7 @@ impl AuthApplicationService { if changed.is_empty() && ui_prefs_patch.is_none() { // No-op — return the current user without a DB write. - return Ok(UserDto::from(user)); + return Ok(PublicUserDto::new(user, false)); } // Persist the typed-field changes first (if any). Skip the @@ -2831,11 +2931,11 @@ impl AuthApplicationService { // Refetch so the returned DTO reflects the merged JSONB bag // (the in-memory `user` above holds the pre-merge value). let refreshed = self.user_storage.get_user_by_id(caller_id).await?; - Ok(UserDto::from(refreshed)) + Ok(PublicUserDto::new(refreshed, false)) } // Alias for consistency with handler method - pub async fn get_user_by_id(&self, user_id: Uuid) -> Result { + pub async fn get_user_by_id(&self, user_id: Uuid) -> Result { self.get_user(user_id).await } @@ -2852,6 +2952,26 @@ impl AuthApplicationService { UserStoragePort::get_user_by_id(&*self.user_storage, user_id).await } + /// Load the full `User` entity + `UserDerivedFlags` for the given + /// id in ONE query. See + /// [`UserRepository::get_user_with_derived_flags`](crate::domain::repositories::user_repository::UserRepository::get_user_with_derived_flags) + /// for the shape and the SELECT that drives it. Used by + /// `/api/auth/me` to build a `SelfUserDto` and by future admin + /// single-user views to build a `FullUserDto` without paying two + /// round-trips. + pub async fn get_user_with_derived_flags( + &self, + user_id: Uuid, + ) -> Result< + ( + crate::domain::entities::user::User, + crate::domain::repositories::user_repository::UserDerivedFlags, + ), + DomainError, + > { + UserStoragePort::get_user_with_derived_flags(&*self.user_storage, user_id).await + } + /// Login-style identifier lookup: dispatches on `@` in the input /// (email path when present, username path when not), identical /// to `login()`'s dispatch. Exposed so the OPAQUE login handler @@ -2908,12 +3028,23 @@ impl AuthApplicationService { target_id: Uuid, expose_system_users: bool, pool: &sqlx::PgPool, - ) -> Result { + ) -> Result { // (1) Self — a single fetch suffices (the check compares the input // UUIDs, so the target read is never needed on this path). + // + // Both branches use `get_user_with_derived_flags` (not the narrow + // `get_user_by_id`) so `PublicUserDto.is_online` on the wire + // reflects the same EXISTS subquery the admin list uses. Without + // this the FE presence dot would only light up on list-derived + // paths (admin seed); single fetches from share pickers / group + // members would show every user as offline regardless of real + // state. See `docs/plan/userdto-refactor.md`. if caller_id == target_id { - let caller = self.user_storage.get_user_by_id(caller_id).await?; - return Ok(UserDto::from(caller)); + let (caller, flags) = self + .user_storage + .get_user_with_derived_flags(caller_id) + .await?; + return Ok(PublicUserDto::new(caller, flags.is_online)); } // Caller and target are independent point reads (the self-case already @@ -2921,16 +3052,26 @@ impl AuthApplicationService { // overlap them with `join!` instead of two serial round-trips. // `caller_res?` first preserves the caller-error precedence of the old // sequential form. (benches/ROUND23.md §P1) + // + // `caller` uses the narrow `get_user_by_id` because we only read + // `is_external()` off it for the visibility gate; nothing about + // the caller ships on the wire. Only `target` needs the wider + // projection. let (caller_res, target_res) = tokio::join!( self.user_storage.get_user_by_id(caller_id), - self.user_storage.get_user_by_id(target_id) + self.user_storage.get_user_with_derived_flags(target_id) ); let caller = caller_res?; // Anti-enumeration: NotFound for everything that doesn't pass. // Convert a real NotFound on `target` to the same anonymous 404, // so existence isn't leaked through differential responses. - let target = match target_res { + // + // Destructure the (User, UserDerivedFlags) tuple immediately so + // `target` keeps its historical `User` shape (accessors still + // work below); the flags come along as `target_flags` for the + // `is_online` propagation into the returned `PublicUserDto`. + let (target, target_flags) = match target_res { Ok(u) => u, Err(e) if e.kind == ErrorKind::NotFound => { tracing::info!( @@ -2974,7 +3115,7 @@ impl AuthApplicationService { })?; if related.is_some() { - return Ok(UserDto::from(target)); + return Ok(PublicUserDto::new(target, target_flags.is_online)); } // (3) External callers stop here — no directory enumeration. @@ -3002,12 +3143,12 @@ impl AuthApplicationService { // (4) Internal target + system-address-book exposed: already public. if !target.is_external() && expose_system_users { - return Ok(UserDto::from(target)); + return Ok(PublicUserDto::new(target, target_flags.is_online)); } // (5) Admin caller: always visible. if caller.role() == UserRole::Admin { - return Ok(UserDto::from(target)); + return Ok(PublicUserDto::new(target, target_flags.is_online)); } // (6) No relationship — anti-enumeration NotFound. @@ -3060,7 +3201,7 @@ impl AuthApplicationService { username: &str, expose_system_users: bool, pool: &sqlx::PgPool, - ) -> Result { + ) -> Result { let target = match self.user_storage.get_user_by_username(username).await { Ok(u) => u, Err(e) if e.kind == ErrorKind::NotFound => { @@ -3087,9 +3228,9 @@ impl AuthApplicationService { } // New method to get user by username - needed for admin user handling - pub async fn get_user_by_username(&self, username: &str) -> Result { + pub async fn get_user_by_username(&self, username: &str) -> Result { let user = self.user_storage.get_user_by_username(username).await?; - Ok(UserDto::from(user)) + Ok(PublicUserDto::new(user, false)) } // Method to count how many admin users exist in the system @@ -3106,42 +3247,46 @@ impl AuthApplicationService { /// out so that internal-user surfaces — system address book, OCS /// sharee search, etc. — never expose external identities. Admin /// surfaces that need the full list should call - /// [`list_users_including_external_with_perms`] instead. - pub async fn list_users(&self, limit: i64, offset: i64) -> Result, DomainError> { - let users = self.user_storage.list_users(limit, offset, false).await?; - Ok(users.into_iter().map(UserDto::from).collect()) - } - - /// Admin-only: lists users including external (grant-only) recipients. - /// Used by the admin user-management UI. - pub async fn list_users_including_external_with_perms( + /// [`list_user_summaries_including_external_with_perms`] instead. + pub async fn list_users( &self, - authorization: &A, - caller_id: Uuid, limit: i64, offset: i64, - ) -> Result, DomainError> { - self.require_admin_caller(authorization, caller_id).await?; - let users = self.user_storage.list_users(limit, offset, true).await?; - Ok(users.into_iter().map(UserDto::from).collect()) + ) -> Result, DomainError> { + let users = self.user_storage.list_users(limit, offset, false).await?; + Ok(users + .into_iter() + .map(|u| PublicUserDto::new(u, false)) + .collect()) } - /// Admin-only compact listing. The detail endpoint retains the complete - /// [`UserDto`]; this path projects only what the management table renders so - /// PostgreSQL never detoasts or transfers avatars/preferences for a page. + /// Admin-only user listing. Returns `Vec` — same + /// `FullUserDto` shape [`SelfUserDto`] embeds, so the FE reads + /// admin table rows and `/me` responses through identical field + /// paths. Includes the avatar (`user.image`) and presence + /// (`user.is_online`) so the admin table renders the vignette + + /// green dot without per-row follow-up fetches to + /// `/api/users/{id}` (the N+1 that motivated the widening — see + /// `docs/plan/userdto-refactor.md` § N+1). This is the sole + /// admin-visible listing path; the former flat + /// `list_users_including_external_with_perms` variant was + /// retired when `?summary` was dropped. pub async fn list_user_summaries_including_external_with_perms( &self, authorization: &A, caller_id: Uuid, limit: i64, offset: i64, - ) -> Result, DomainError> { + ) -> Result, DomainError> { self.require_admin_caller(authorization, caller_id).await?; - let users = self + let rows = self .user_storage - .list_user_summaries(limit, offset, true) + .list_users_with_derived_flags(limit, offset, true) .await?; - Ok(users.into_iter().map(AdminUserSummaryDto::from).collect()) + Ok(rows + .into_iter() + .map(|(user, flags)| FullUserDto::build(user, flags)) + .collect()) } /// Service-layer gate for administrator-scoped user-directory operations. @@ -3164,9 +3309,16 @@ impl AuthApplicationService { } /// Searches internal users only. See [`list_users`] for the rationale. - pub async fn search_users(&self, query: &str, limit: i64) -> Result, DomainError> { + pub async fn search_users( + &self, + query: &str, + limit: i64, + ) -> Result, DomainError> { let users = self.user_storage.search_users(query, limit, false).await?; - Ok(users.into_iter().map(UserDto::from).collect()) + Ok(users + .into_iter() + .map(|u| PublicUserDto::new(u, false)) + .collect()) } /// Username-only search for the NC sharee autocomplete: identical @@ -3196,8 +3348,8 @@ impl AuthApplicationService { // `interfaces/api/routes.rs::admin_router`) — but every admin // method here still calls `require_admin_caller` as a // defense-in-depth check, matching the pattern - // `list_users_including_external_with_perms` established. If a - // handler is ever wired outside the /admin subtree, the AuthZ + // `list_user_summaries_including_external_with_perms` established. + // If a handler is ever wired outside the /admin subtree, the AuthZ // still holds. /// List sessions for the admin panel. `user_id_filter = Some(uuid)` @@ -3272,7 +3424,7 @@ impl AuthApplicationService { pub async fn admin_create_user( &self, dto: crate::application::dtos::settings_dto::AdminCreateUserDto, - ) -> Result { + ) -> Result { // Validate username length if dto.username.len() < 3 || dto.username.len() > 254 { return Err(DomainError::new( @@ -3430,7 +3582,16 @@ impl AuthApplicationService { created.id(), created.is_external() ); - Ok(UserDto::from(created)) + // Return `FullUserDto` — same shape as `GET /api/admin/users/{id}` + // and one row of the admin list. Admin surfaces uniformly return + // FullUserDto so the SPA / test asserts don't need to know which + // admin endpoint they came from. Fresh user has no session yet + // (`is_online = false`) and no OPAQUE registration; `has_password` + // reflects whatever the admin passed in the DTO. + let created_id = created.id(); + let (user, flags) = + UserStoragePort::get_user_with_derived_flags(&*self.user_storage, created_id).await?; + Ok(FullUserDto::build(user, flags)) } /// Admin-only: reset a user's password. @@ -3526,10 +3687,20 @@ impl AuthApplicationService { Ok(()) } - /// Get a single user by ID (for admin panel) - pub async fn get_user_admin(&self, user_id: Uuid) -> Result { - let user = self.user_storage.get_user_by_id(user_id).await?; - Ok(UserDto::from(user)) + /// Get a single user by ID (for admin panel). + /// + /// Returns `FullUserDto` — same shape as one row of + /// `/api/admin/users` — so admin single-user views (detail modal, + /// per-user edit page) render the same fields the list surfaces. + /// The single-row admin view is the canonical observation surface + /// for admin-visible signals like `email_verified_at` / + /// `has_password` / `opaque_registered` / `last_login_at` — none + /// of which live on the peer-view `PublicUserDto`. See + /// `docs/plan/userdto-refactor.md`. + pub async fn get_user_admin(&self, user_id: Uuid) -> Result { + let (user, flags) = + UserStoragePort::get_user_with_derived_flags(&*self.user_storage, user_id).await?; + Ok(FullUserDto::build(user, flags)) } /// Delete a user by ID (admin only). @@ -4608,8 +4779,8 @@ impl AuthApplicationService { // through the browser's redirect chain. Session is minted // unbound; the SPA calls `POST /api/auth/dpop/bind` post- // redirect to bind it (see Gate 3). Token accordingly ships - // without `cnf.jkt`. - let access_token = self.token_service.generate_access_token(&user, None)?; + // without `cnf.jkt`. Session constructed first so its id + // feeds the token's `sid` claim. let refresh_token = self.token_service.generate_refresh_token(); let mut session = Session::new( @@ -4630,11 +4801,22 @@ impl AuthApplicationService { if let Some(sid) = claims.sid.as_ref() { session = session.with_oidc_sid(sid.clone()); } + // Mint AFTER session is fully configured so `sid` claim + // aligns with the row about to be inserted. + let access_token = + self.token_service + .generate_access_token(&user, Some(session.id()), None)?; + // Build the SelfUserDto before `session` is consumed by the + // storage call — the builder reads `session.dpop_jkt()` + // (None here since OIDC callbacks land unbound and the SPA + // finishes binding post-redirect). + let user_id = user.id(); + let user_dto = self.build_self_user_dto(user_id, &session).await?; self.session_storage.create_session(session).await?; - let force_password_change = self.read_force_password_change(user.id()).await; + let force_password_change = self.read_force_password_change(user_id).await; let auth_response = AuthResponseDto { - user: UserDto::from(user), + user: user_dto, access_token, refresh_token, token_type: "Bearer".to_string(), diff --git a/src/application/services/device_auth_service.rs b/src/application/services/device_auth_service.rs index 8e0a455e..8e10751e 100644 --- a/src/application/services/device_auth_service.rs +++ b/src/application/services/device_auth_service.rs @@ -180,10 +180,12 @@ impl DeviceAuthService { // clients that don't run WebCrypto — always unbound (`None` // for the `dpop_jkt` param), which the DPoP middleware exempts // from proof requirements. See `docs/plan/dpop.md` Gate 9. - let access_token = self.token_service.generate_access_token(&user, None)?; + // + // Session constructed first so its id feeds the token's + // `sid` claim — the auth middleware uses this to stamp + // per-session liveness without a DB round trip + // (`docs/plan/sessions.md`). let refresh_token = self.token_service.generate_refresh_token(); - - // Persist refresh token as a session let session = Session::new( user_id, refresh_token.clone(), @@ -193,6 +195,9 @@ impl DeviceAuthService { Uuid::new_v4(), crate::domain::entities::session::SessionOrigin::Device, ); + let access_token = + self.token_service + .generate_access_token(&user, Some(session.id()), None)?; self.session_storage.create_session(session).await?; // Store tokens on the device code entity diff --git a/src/application/services/file_retrieval_service.rs b/src/application/services/file_retrieval_service.rs index ac253662..6bd37ea9 100644 --- a/src/application/services/file_retrieval_service.rs +++ b/src/application/services/file_retrieval_service.rs @@ -274,9 +274,15 @@ impl FileRetrievalService { } /// Try to transcode image content to WebP and return transcoded variant. + /// + /// `source_hash` keys the durable derived tier. The caller has it as + /// `dto.content_hash`; passing it rather than hashing here matters, + /// because hashing would be a BLAKE3 over the whole file on every + /// request that reaches this path. async fn try_transcode( &self, id: &str, + source_hash: Option<&str>, content: &Bytes, mime: &str, file_size: u64, @@ -291,7 +297,7 @@ impl FileRetrievalService { } let format = OutputFormat::WebP; match transcode - .get_transcoded(id, content.clone(), mime, format) + .get_transcoded(id, source_hash, content.clone(), mime, format) .await { Ok((transcoded, webp_mime, true)) => { @@ -364,7 +370,16 @@ impl FileRetrievalService { if do_transcode && let Some((t, m)) = self - .try_transcode(id, &content_bytes, &mime_type, file_size, true) + .try_transcode( + id, + // Empty on the hash-less stub DTOs (external mounts), + // which the content-keyed tier cannot serve anyway. + Some(&*dto.content_hash).filter(|h: &&str| !h.is_empty()), + &content_bytes, + &mime_type, + file_size, + true, + ) .await { return Ok(( diff --git a/src/application/services/storage_usage_service.rs b/src/application/services/storage_usage_service.rs index 6ef66d36..392e22ee 100644 --- a/src/application/services/storage_usage_service.rs +++ b/src/application/services/storage_usage_service.rs @@ -578,7 +578,7 @@ impl StorageUsageService { pub const USAGE_RECONCILE_JOB_NAME: &str = "usage_reconcile"; -use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs}; +use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs, Mutates}; use async_trait::async_trait; impl StorageUsageService { @@ -603,6 +603,19 @@ impl JobHandler for StorageUsageService { USAGE_RECONCILE_JOB_NAME } + fn description(&self) -> &'static str { + "Recomputes the cached storage counters from the underlying file \ + sizes — drives first, then the per-user envelope derived from \ + them — and corrects any that drifted. This is the corrective \ + counterpart to drives_consistency, which only reports the drift." + } + + /// Rewrites the counters it finds wrong. Safe to trigger: it recomputes + /// from the files themselves, so a run is idempotent. + fn mutates(&self) -> Mutates { + Mutates::Always + } + /// Runs both reconciliation sweeps — drives first, then users — /// and reports the total number of rows corrected. /// diff --git a/src/bin/migrate-nfc-filenames.rs b/src/bin/migrate-nfc-filenames.rs deleted file mode 100644 index 270eddb7..00000000 --- a/src/bin/migrate-nfc-filenames.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! `migrate-nfc-filenames` — one-shot CLI to NFC-normalize -//! `storage.files.name` across an OxiCloud instance. -//! -//! Why: PostgreSQL compares bytes literally and the `UNIQUE` -//! index on `(folder_id, name, user_id) WHERE NOT is_trashed` -//! does not catch Unicode normalization differences. macOS APFS -//! stores filenames in NFD; browsers post NFC. A file uploaded -//! from the web ("café.txt", NFC) and the same name re-uploaded -//! from a NextCloud desktop client on macOS (round-tripped to -//! NFD: `e` + combining acute) lands as two distinct rows, both -//! visible in the listing, both pointing at the same blob. -//! -//! What this does: -//! -//! 1. Scans every non-trashed file row. -//! 2. For each row whose name ≠ NFC(name): -//! - If no other row in the same `(folder_id, user_id)` already -//! holds the NFC form → UPDATE the row's name to NFC. -//! - If a collision exists with **same blob_hash**: trash the -//! newer of the two (`is_trashed = true`, `trashed_at = NOW()`). -//! User can restore from the trash UI if needed. -//! - If a collision exists with **different blob_hash**: rename -//! the newer row to `{nfc_name}.duplicate`, incrementing the -//! suffix (`.duplicate-1`, `.duplicate-2`, …) until a free name -//! is found. Preserves both files; user can inspect and resolve. -//! - In both collision cases, the surviving (older) row's name -//! is also normalized to NFC. -//! -//! Run: -//! `cargo run --bin migrate-nfc-filenames -- --dry-run` -//! `cargo run --bin migrate-nfc-filenames` -//! -//! Folder rows are NOT touched in this pass — trashing a folder -//! affects descendants; that pass is deferred to a follow-up. - -use chrono::{DateTime, Utc}; -use sqlx::{PgPool, Row}; -use std::env; -use uuid::Uuid; - -use oxicloud::domain::services::path_service::normalize_storage_name; - -#[derive(Debug, Clone)] -struct FileRow { - id: Uuid, - folder_id: Option, - user_id: Uuid, - name: String, - blob_hash: String, - created_at: DateTime, -} - -#[derive(Default)] -struct Stats { - scanned: u64, - already_nfc: u64, - normalized_in_place: u64, - deduped_same_content: u64, - renamed_duplicate: u64, -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let args: Vec = env::args().collect(); - let dry_run = args.iter().any(|a| a == "--dry-run"); - - let database_url = - env::var("DATABASE_URL").expect("DATABASE_URL must be set in the environment"); - - let pool = PgPool::connect(&database_url).await?; - - println!( - "=== NFC filename migration ({}) ===", - if dry_run { - "DRY RUN — no writes" - } else { - "EXECUTING" - } - ); - println!(); - - let rows = load_non_trashed_files(&pool).await?; - println!("Loaded {} non-trashed file rows", rows.len()); - println!(); - - let mut stats = Stats { - scanned: rows.len() as u64, - ..Default::default() - }; - - for row in &rows { - let nfc_name = normalize_storage_name(&row.name); - if nfc_name == row.name { - stats.already_nfc += 1; - continue; - } - - // Row is in non-NFC form. Look for a collision in the same - // (folder_id, user_id) scope, including rows that may also - // be non-NFC but happen to normalize to the same NFC value. - let collision = find_collision(&pool, row, &nfc_name).await?; - - match collision { - None => { - println!( - "NORMALIZE {} user={} '{}' → '{}'", - row.id, row.user_id, row.name, nfc_name - ); - if !dry_run { - sqlx::query("UPDATE storage.files SET name = $1 WHERE id = $2") - .bind(&nfc_name) - .bind(row.id) - .execute(&pool) - .await?; - } - stats.normalized_in_place += 1; - } - Some(other) => { - // Pick winner/loser by `created_at` — older wins. - let (older, newer) = if row.created_at <= other.created_at { - (row, &other) - } else { - (&other, row) - }; - - if older.blob_hash == newer.blob_hash { - // Same content → trash the newer; promote older's - // name to NFC if it isn't already. - println!( - "DEDUP newer={} (trash, same blob) older={} user={} hash={}", - newer.id, - older.id, - older.user_id, - &older.blob_hash[..16.min(older.blob_hash.len())] - ); - if !dry_run { - sqlx::query( - "UPDATE storage.files - SET is_trashed = TRUE, - trashed_at = NOW() - WHERE id = $1", - ) - .bind(newer.id) - .execute(&pool) - .await?; - normalize_survivor_name(&pool, older, &nfc_name).await?; - } - stats.deduped_same_content += 1; - } else { - // Different content → rename newer to a free - // `{nfc_name}.duplicate[-N]`; promote older to NFC. - let disambiguated = find_free_duplicate_name(&pool, newer, &nfc_name).await?; - println!( - "RENAME newer={} (different blob) older={} '{}' → '{}'", - newer.id, older.id, newer.name, disambiguated - ); - if !dry_run { - sqlx::query("UPDATE storage.files SET name = $1 WHERE id = $2") - .bind(&disambiguated) - .bind(newer.id) - .execute(&pool) - .await?; - normalize_survivor_name(&pool, older, &nfc_name).await?; - } - stats.renamed_duplicate += 1; - } - } - } - } - - println!(); - println!("=== Summary ==="); - println!(" scanned : {}", stats.scanned); - println!( - " already in NFC : {}", - stats.already_nfc - ); - println!( - " normalized in place (no collision) : {}", - stats.normalized_in_place - ); - println!( - " dedup-trashed (same content) : {}", - stats.deduped_same_content - ); - println!( - " renamed to .duplicate : {}", - stats.renamed_duplicate - ); - if dry_run { - println!(); - println!("DRY RUN — no rows were written. Re-run without --dry-run to apply."); - } - - Ok(()) -} - -async fn load_non_trashed_files(pool: &PgPool) -> Result, Box> { - let raw = sqlx::query( - "SELECT id, folder_id, user_id, name, blob_hash, created_at - FROM storage.files - WHERE NOT is_trashed - ORDER BY created_at", - ) - .fetch_all(pool) - .await?; - - let mut out = Vec::with_capacity(raw.len()); - for r in raw { - out.push(FileRow { - id: r.try_get("id")?, - folder_id: r.try_get("folder_id")?, - user_id: r.try_get("user_id")?, - name: r.try_get("name")?, - blob_hash: r.try_get("blob_hash")?, - created_at: r.try_get("created_at")?, - }); - } - Ok(out) -} - -/// Looks for a row in the same `(folder_id, user_id)` scope whose -/// CURRENT name equals `nfc_name`, excluding the row being processed. -/// The other row may itself be in non-NFC form whose normalized -/// representation happens to differ from `nfc_name`; the collision -/// check is intentionally based on stored bytes (matching the -/// UNIQUE-index semantics that this migration is repairing). -async fn find_collision( - pool: &PgPool, - row: &FileRow, - nfc_name: &str, -) -> Result, Box> { - let result = sqlx::query( - "SELECT id, folder_id, user_id, name, blob_hash, created_at - FROM storage.files - WHERE name = $1 - AND user_id = $2 - AND ($3::uuid IS NULL AND folder_id IS NULL - OR folder_id = $3::uuid) - AND id <> $4 - AND NOT is_trashed - LIMIT 1", - ) - .bind(nfc_name) - .bind(row.user_id) - .bind(row.folder_id) - .bind(row.id) - .fetch_optional(pool) - .await?; - - Ok(result.map(|r| FileRow { - id: r.get("id"), - folder_id: r.get("folder_id"), - user_id: r.get("user_id"), - name: r.get("name"), - blob_hash: r.get("blob_hash"), - created_at: r.get("created_at"), - })) -} - -/// Finds a free name in the form `{nfc_name}.duplicate` or -/// `{nfc_name}.duplicate-N` for `N >= 1`, scoped to the row's -/// `(folder_id, user_id)`. Returns the first candidate that does -/// not currently exist as a non-trashed row. -async fn find_free_duplicate_name( - pool: &PgPool, - row: &FileRow, - nfc_name: &str, -) -> Result> { - let mut suffix: u32 = 0; - loop { - let candidate = if suffix == 0 { - format!("{}.duplicate", nfc_name) - } else { - format!("{}.duplicate-{}", nfc_name, suffix) - }; - - let taken: bool = sqlx::query_scalar( - "SELECT EXISTS( - SELECT 1 FROM storage.files - WHERE name = $1 - AND user_id = $2 - AND ($3::uuid IS NULL AND folder_id IS NULL - OR folder_id = $3::uuid) - AND id <> $4 - AND NOT is_trashed)", - ) - .bind(&candidate) - .bind(row.user_id) - .bind(row.folder_id) - .bind(row.id) - .fetch_one(pool) - .await?; - - if !taken { - return Ok(candidate); - } - suffix = suffix.saturating_add(1); - // Safety bound — should never trigger under realistic data. - if suffix > 10_000 { - return Err(format!( - "Exhausted .duplicate-N suffixes for '{}' in scope (user={}, folder_id={:?})", - nfc_name, row.user_id, row.folder_id - ) - .into()); - } - } -} - -/// If the surviving (older) row's stored name is not yet in NFC, -/// UPDATE it now that the collision has been resolved. -async fn normalize_survivor_name( - pool: &PgPool, - survivor: &FileRow, - nfc_name: &str, -) -> Result<(), Box> { - if survivor.name == nfc_name { - return Ok(()); - } - sqlx::query("UPDATE storage.files SET name = $1 WHERE id = $2") - .bind(nfc_name) - .bind(survivor.id) - .execute(pool) - .await?; - Ok(()) -} diff --git a/src/bin/oxicloud-cli.rs b/src/bin/oxicloud-cli.rs deleted file mode 100644 index d4b8e135..00000000 --- a/src/bin/oxicloud-cli.rs +++ /dev/null @@ -1,278 +0,0 @@ -//! `oxicloud-cli` — operator toolbox for the OxiCloud deployment. -//! -//! Single binary with subcommand tree, shipped alongside the `oxicloud` -//! server binary. Replaces the per-task one-off bins (previously -//! `opaque-setup`, and any future `opaque-reset` etc.) with a -//! discoverable `--help`-driven surface so the container ships one -//! toolbox binary rather than N one-off ones. -//! -//! ## Layout -//! -//! ```text -//! oxicloud-cli [flags] -//! -//! Domains: -//! opaque OPAQUE aPAKE substrate management -//! setup Print a fresh ServerSetup value for OXICLOUD_AUTH_OPAQUE_SERVER_SETUP -//! reset Clear envelope(s) so silent-migration re-mints under current KSF -//! ``` -//! -//! Growth pattern: each new domain gets its own module below (e.g. -//! `mod opaque`) with a `#[derive(Subcommand)]` enum for its actions -//! and a `run(args) -> ExitCode` entrypoint. Keep each module -//! self-contained so a future extraction is a file move. -//! -//! ## Environment -//! -//! * `DATABASE_URL` — required by any subcommand that talks to the DB -//! (`opaque reset`); not needed for pure primitive helpers -//! (`opaque setup`). Each subcommand documents its own dependencies. - -use std::process::ExitCode; - -use clap::{Parser, Subcommand}; - -#[derive(Parser)] -#[command( - name = "oxicloud-cli", - version, - about = "OxiCloud operator toolbox", - long_about = "OxiCloud operator toolbox — subcommand entrypoint for operational \ - tasks that don't belong in the main server binary." -)] -struct Cli { - #[command(subcommand)] - domain: Domain, -} - -#[derive(Subcommand)] -enum Domain { - /// OPAQUE aPAKE substrate management (setup, reset). - Opaque { - #[command(subcommand)] - action: opaque::Action, - }, -} - -#[tokio::main(flavor = "current_thread")] -async fn main() -> ExitCode { - let cli = Cli::parse(); - match cli.domain { - Domain::Opaque { action } => opaque::run(action).await, - } -} - -// ── opaque domain ────────────────────────────────────────────────────── - -mod opaque { - use std::env; - use std::process::ExitCode; - - use clap::Subcommand; - use oxicloud::infrastructure::services::opaque_service::OpaqueService; - use sqlx::{PgPool, Row}; - - #[derive(Subcommand)] - pub enum Action { - /// Generate a fresh OPAQUE ServerSetup and print its base64 - /// encoding to stdout. Guidance goes to stderr so shell - /// pipelines capture cleanly. - /// - /// Run ONCE per deployment; persist the printed value as - /// `OXICLOUD_AUTH_OPAQUE_SERVER_SETUP`. Rotating this value - /// invalidates every user's OPAQUE registration — treat it - /// like your JWT secret. - Setup, - - /// Clear the OPAQUE envelope for one user or all users - /// WITHOUT touching password or setting force_password_change. - /// - /// Use case: KSF rotation. If you change - /// OXICLOUD_AUTH_OPAQUE_KSF_* values, existing envelopes - /// become cryptographically incompatible with the newly - /// published KSF — logins fail with InvalidCredentials. - /// Nulling the envelope columns forces the SPA's `/lookup` - /// to report `hasOpaque: false`, which routes the next login - /// through legacy `/api/auth/login`; silent-migration then - /// mints a fresh envelope under the CURRENT KSF. Passwords - /// are unchanged. - /// - /// NOT for forgotten-passphrase recovery — use the admin - /// password-reset endpoint (`PUT /api/admin/users/{id}/password`) - /// which sets a temp password + force_change flag in one shot. - Reset { - /// Email OR username to reset (dispatched on `@` presence, - /// same rule as `POST /api/auth/login`). - #[arg(long, conflicts_with = "all")] - user: Option, - - /// Reset every user with an OPAQUE envelope. - #[arg(long, conflicts_with = "user")] - all: bool, - - /// Print what would change without touching the DB. - #[arg(long)] - dry_run: bool, - }, - } - - pub async fn run(action: Action) -> ExitCode { - match action { - Action::Setup => run_setup(), - Action::Reset { user, all, dry_run } => run_reset(user, all, dry_run).await, - } - } - - fn run_setup() -> ExitCode { - // Match the legacy `opaque-setup` bin's contract: - // - value on stdout, no trailing commentary (pipeline-safe) - // - guidance on stderr - let b64 = OpaqueService::generate_server_setup_b64(); - println!("{b64}"); - eprintln!(); - eprintln!("=== OPAQUE server setup generated. ==="); - eprintln!("Persist the line above in OXICLOUD_AUTH_OPAQUE_SERVER_SETUP."); - eprintln!("NEVER rotate: rotating invalidates every user's registration."); - eprintln!("Treat this value like your JWT secret."); - ExitCode::from(0) - } - - async fn run_reset(user: Option, all: bool, dry_run: bool) -> ExitCode { - // clap enforces `conflicts_with`, but not "at least one of". - // Belt-and-braces check here so the failure is explicit. - if user.is_none() && !all { - eprintln!("opaque reset: pass either --user or --all"); - return ExitCode::from(2); - } - - let database_url = match env::var("DATABASE_URL") { - Ok(v) => v, - Err(_) => { - eprintln!("opaque reset: DATABASE_URL not set"); - return ExitCode::from(2); - } - }; - let pool = match PgPool::connect(&database_url).await { - Ok(p) => p, - Err(e) => { - eprintln!("opaque reset: failed to connect to database: {e}"); - return ExitCode::from(1); - } - }; - - // Preview the affected row set before writing. Doubles as - // dry-run output and as diagnostics when --user matches nothing. - // Envelope-presence bool lets the operator see which rows had - // an envelope vs which only carry a stale migration mark. - let select_sql = if all { - r#" - SELECT id, email, (opaque_envelope IS NOT NULL) AS had_envelope - FROM auth.users - WHERE opaque_envelope IS NOT NULL - OR opaque_migrated_at IS NOT NULL - ORDER BY email - "# - } else { - r#" - SELECT id, email, (opaque_envelope IS NOT NULL) AS had_envelope - FROM auth.users - WHERE CASE WHEN $1 LIKE '%@%' THEN email = $1 ELSE username = $1 END - "# - }; - let rows_result = if all { - sqlx::query(select_sql).fetch_all(&pool).await - } else { - let ident = user.as_deref().unwrap(); - sqlx::query(select_sql).bind(ident).fetch_all(&pool).await - }; - let rows = match rows_result { - Ok(r) => r, - Err(e) => { - eprintln!("opaque reset: query failed: {e}"); - return ExitCode::from(1); - } - }; - if rows.is_empty() { - if all { - println!("opaque reset: no users have an OPAQUE envelope — nothing to do."); - return ExitCode::from(0); - } else { - eprintln!( - "opaque reset: no user matches --user {} — nothing changed.", - user.as_deref().unwrap_or("") - ); - return ExitCode::from(1); - } - } - - println!( - "opaque reset ({}): {} row(s) to affect", - if dry_run { - "DRY RUN — no writes" - } else { - "EXECUTING" - }, - rows.len() - ); - for row in &rows { - let id: uuid::Uuid = row.get("id"); - let email: String = row.get("email"); - let had_envelope: bool = row.get("had_envelope"); - println!( - " {} {} {}", - id, - email, - if had_envelope { - "had-envelope" - } else { - "no-envelope-had-migrated-mark" - } - ); - } - if dry_run { - return ExitCode::from(0); - } - - // Actual UPDATE. Kept identical in shape to the SELECT above so - // the planner sees the same query pattern for both. We - // DELIBERATELY do NOT touch password_hash or - // force_password_change_at_next_login — this tool is scoped - // to "the passwords are fine, the envelopes are stale." - let update_sql_all = r#" - UPDATE auth.users - SET opaque_envelope = NULL, - opaque_ciphersuite_version = NULL, - opaque_registered_at = NULL, - opaque_migrated_at = NULL - WHERE opaque_envelope IS NOT NULL - OR opaque_migrated_at IS NOT NULL - "#; - let update_sql_one = r#" - UPDATE auth.users - SET opaque_envelope = NULL, - opaque_ciphersuite_version = NULL, - opaque_registered_at = NULL, - opaque_migrated_at = NULL - WHERE CASE WHEN $1 LIKE '%@%' THEN email = $1 ELSE username = $1 END - "#; - let write_result = if all { - sqlx::query(update_sql_all).execute(&pool).await - } else { - let ident = user.as_deref().unwrap(); - sqlx::query(update_sql_one).bind(ident).execute(&pool).await - }; - let affected = match write_result { - Ok(r) => r.rows_affected(), - Err(e) => { - eprintln!("opaque reset: update failed: {e}"); - return ExitCode::from(1); - } - }; - println!( - "opaque reset: cleared envelope columns on {affected} row(s). \ - Users log in with their existing password; silent-migration \ - re-mints envelopes under the current KSF on next login." - ); - ExitCode::from(0) - } -} diff --git a/src/cli/migrate.rs b/src/cli/migrate.rs new file mode 100644 index 00000000..89f6e06a --- /dev/null +++ b/src/cli/migrate.rs @@ -0,0 +1,760 @@ +//! `migrate` subcommand domain — one-time data migrations. +//! +//! Sqlx schema migrations run automatically at boot via +//! `sqlx::migrate!()` — this domain is reserved for **data** migrations +//! that need explicit operator invocation (data-loss ambiguity, long +//! runtime, or historical schema-drift cleanup). +//! +//! Currently ships one action: `nfc-filenames` — cleans up NFD/NFC +//! name collisions in databases populated before the write-time fix +//! landed at the repository layer (see +//! `folder_db_repository::create_folder`, `file_blob_write_repository` +//! ingest paths, `drive_pg_repository::create_shared_drive_atomic`). +//! New installs get NFC on every ingest and never accumulate drift. +//! +//! Covers BOTH `storage.files.name` and `storage.folders.name`. The +//! folder pass was added 2026-09-04 in response to +//! AtalayaLabs/OxiCloud#706 (macOS Finder folder upload landed NFD; +//! the file-only migrate did nothing for the reporter). Folders have +//! no `blob_hash`, so the collision branch is "older keeps NFC name, +//! newer becomes `.duplicate[-N]`" only — no dedup-by-trash arm, +//! because trashing a folder strands its subtree. +//! +//! Previously lived in a standalone `migrate-nfc-filenames` binary +//! before the v0.9.0 CLI/server merge — see docs/plan/bundled-binary.md § 1b. +//! The 149-line body of `main()` moved here as `run_nfc_filenames()` +//! with `env::args()` parsing replaced by clap. +//! +//! **Retention: indefinite.** An earlier version of this doc set a +//! "future removal target: v1.0" — retracted 2026-09-04 for three +//! reasons: +//! +//! 1. The pre-2026-09 write-side normalization was DEAD CODE +//! (invariants at `File::new` / `Folder::new_folder` entity +//! constructors that the create path bypassed), so every +//! OxiCloud version shipped before that date accumulated NFD +//! content and has a real remediation need. Many self-hosters +//! won't upgrade for months. +//! 2. Prior versions of THIS migrate command referenced the D7- +//! dropped `user_id` column and errored on first run, so users +//! who tried to apply it never got anywhere. The 2026-09-04 fix +//! makes it work again — but re-applying to instances that were +//! "already migrated" (they weren't) is now the only remediation +//! path for their historical NFD content. +//! 3. Post-fix installs run it as a no-op (all `already_nfc`), so +//! the cost of shipping it forever is zero and the safety it +//! offers for late-upgraders is real. + +use std::env; + +use chrono::{DateTime, Utc}; +use clap::Subcommand; +use sqlx::{PgPool, Row}; +use uuid::Uuid; + +use crate::domain::services::path_service::normalize_storage_name; + +#[derive(Subcommand)] +pub enum Action { + /// NFC-normalize `storage.files.name` AND `storage.folders.name` + /// across the instance. + /// + /// Historical cleanup for databases with rows written before the + /// repo-level write-time normalization landed (see module doc for + /// the exact repo methods). Post-fix installs run this as a + /// harmless no-op — every row reports `already_nfc`. + /// + /// Collision handling (files): + /// * No collision → UPDATE row name to NFC. + /// * Same blob content → trash the newer row. + /// * Different content → rename the newer to `{name}.duplicate[-N]`. + /// + /// Collision handling (folders): + /// * No collision → UPDATE row name to NFC. + /// * Collision → rename the newer to `{name}.duplicate[-N]`; the + /// dedup-by-trash arm from the file path is deliberately absent + /// because trashing a folder strands its subtree. + /// + /// In all collision cases, the surviving (older) row's name is + /// also normalized to NFC. + NfcFilenames { + /// Print what would change without touching the DB. + #[arg(long)] + dry_run: bool, + }, +} + +pub async fn run(action: Action) -> u8 { + match action { + Action::NfcFilenames { dry_run } => run_nfc_filenames(dry_run).await, + } +} + +#[derive(Debug, Clone)] +struct FileRow { + id: Uuid, + folder_id: Option, + /// §14 provenance — the user who created the row. Pre-D7 this + /// lived on `user_id`; post-D7 it's `created_by` and `user_id` + /// no longer exists. Not part of the collision scope (the DB + /// unique index is `(folder_id, name) WHERE NOT is_trashed` — + /// no user column in it), but surfaced in the log lines so an + /// operator triaging a large migration output can spot rows + /// owned by a specific principal without a separate query. + created_by: Option, + name: String, + blob_hash: String, + created_at: DateTime, +} + +/// Structural sibling of [`FileRow`] for `storage.folders`. Folders +/// have no `blob_hash` — there is no "same content dedup" branch on +/// collision, only "keep older, rename newer to .duplicate". Added to +/// close the AtalayaLabs/OxiCloud#706 recovery gap: pre-fix DBs with +/// NFD-named folders (macOS Finder / NC desktop upload from macOS) +/// were unreachable via NFC-normalizing clients, and the file-only +/// migrate did nothing for them. +#[derive(Debug, Clone)] +struct FolderRow { + id: Uuid, + parent_id: Option, + /// §14 provenance — see [`FileRow::created_by`]. + created_by: Option, + name: String, + created_at: DateTime, +} + +#[derive(Default)] +struct Stats { + scanned: u64, + already_nfc: u64, + normalized_in_place: u64, + deduped_same_content: u64, + renamed_duplicate: u64, + // Folder stats — deliberately separate so operators reading the + // summary see "X files, Y folders" instead of one blended count + // that hides the fact that a run touched both scopes. + folders_scanned: u64, + folders_already_nfc: u64, + folders_normalized_in_place: u64, + folders_renamed_duplicate: u64, +} + +async fn run_nfc_filenames(dry_run: bool) -> u8 { + let database_url = match env::var("DATABASE_URL") { + Ok(v) => v, + Err(_) => { + eprintln!("migrate nfc-filenames: DATABASE_URL not set"); + return 2; + } + }; + + let pool = match PgPool::connect(&database_url).await { + Ok(p) => p, + Err(e) => { + eprintln!("migrate nfc-filenames: failed to connect to database: {e}"); + return 1; + } + }; + + println!( + "=== NFC filename migration ({}) ===", + if dry_run { + "DRY RUN — no writes" + } else { + "EXECUTING" + } + ); + println!(); + + let rows = match load_non_trashed_files(&pool).await { + Ok(r) => r, + Err(e) => { + eprintln!("migrate nfc-filenames: initial scan failed: {e}"); + return 1; + } + }; + println!("Loaded {} non-trashed file rows", rows.len()); + println!(); + + let mut stats = Stats { + scanned: rows.len() as u64, + ..Default::default() + }; + + for row in &rows { + let nfc_name = normalize_storage_name(&row.name); + if nfc_name == row.name { + stats.already_nfc += 1; + continue; + } + + // Row is in non-NFC form. Look for a collision in the same + // folder scope (the DB's unique-index scope for storage.files — + // `(folder_id, name) WHERE NOT is_trashed`), including rows + // that may also be non-NFC but happen to normalize to the same + // NFC value. Pre-D7 this scope included user_id; the column + // has since been dropped (`docs/plan/drive.md` §D7), so the + // scope now matches today's unique constraint verbatim. + let collision = match find_collision(&pool, row, &nfc_name).await { + Ok(c) => c, + Err(e) => { + eprintln!( + "migrate nfc-filenames: collision query failed for {}: {e}", + row.id + ); + return 1; + } + }; + + match collision { + None => { + println!( + "NORMALIZE file={} folder={:?} created_by={:?} '{}' ({}B) → '{}' ({}B)", + row.id, + row.folder_id, + row.created_by, + row.name, + row.name.len(), + nfc_name, + nfc_name.len(), + ); + if !dry_run + && let Err(e) = sqlx::query("UPDATE storage.files SET name = $1 WHERE id = $2") + .bind(&nfc_name) + .bind(row.id) + .execute(&pool) + .await + { + eprintln!("migrate nfc-filenames: rename failed for {}: {e}", row.id); + return 1; + } + stats.normalized_in_place += 1; + } + Some(other) => { + // Pick winner/loser by `created_at` — older wins. + let (older, newer) = if row.created_at <= other.created_at { + (row, &other) + } else { + (&other, row) + }; + + if older.blob_hash == newer.blob_hash { + // Same content → trash the newer; promote older's + // name to NFC if it isn't already. + println!( + "DEDUP newer={} (trash, same blob) older={} folder={:?} created_by={:?} hash={}", + newer.id, + older.id, + older.folder_id, + older.created_by, + &older.blob_hash[..16.min(older.blob_hash.len())] + ); + if !dry_run { + if let Err(e) = sqlx::query( + "UPDATE storage.files + SET is_trashed = TRUE, + trashed_at = NOW() + WHERE id = $1", + ) + .bind(newer.id) + .execute(&pool) + .await + { + eprintln!("migrate nfc-filenames: trash failed for {}: {e}", newer.id); + return 1; + } + if let Err(e) = normalize_survivor_name(&pool, older, &nfc_name).await { + eprintln!( + "migrate nfc-filenames: survivor rename failed for {}: {e}", + older.id + ); + return 1; + } + } + stats.deduped_same_content += 1; + } else { + // Different content → rename newer to a free + // `{nfc_name}.duplicate[-N]`; promote older to NFC. + let disambiguated = match find_free_duplicate_name(&pool, newer, &nfc_name) + .await + { + Ok(n) => n, + Err(e) => { + eprintln!( + "migrate nfc-filenames: duplicate-name search failed for {}: {e}", + newer.id + ); + return 1; + } + }; + println!( + "RENAME newer={} (different blob) older={} created_by={:?} '{}' ({}B) → '{}' ({}B)", + newer.id, + older.id, + newer.created_by, + newer.name, + newer.name.len(), + disambiguated, + disambiguated.len(), + ); + if !dry_run { + if let Err(e) = + sqlx::query("UPDATE storage.files SET name = $1 WHERE id = $2") + .bind(&disambiguated) + .bind(newer.id) + .execute(&pool) + .await + { + eprintln!( + "migrate nfc-filenames: disambiguation rename failed for {}: {e}", + newer.id + ); + return 1; + } + if let Err(e) = normalize_survivor_name(&pool, older, &nfc_name).await { + eprintln!( + "migrate nfc-filenames: survivor rename failed for {}: {e}", + older.id + ); + return 1; + } + } + stats.renamed_duplicate += 1; + } + } + } + } + + // Second pass: folders. Same shape as the file loop but no dedup + // branch (folders have no `blob_hash`). Added to close + // AtalayaLabs/OxiCloud#706 — a reported macOS-Finder folder upload + // with an NFD name was unreachable via NFC-normalizing clients and + // this migration was the operator's documented recovery path. + if let Err(code) = run_folders(&pool, dry_run, &mut stats).await { + return code; + } + + println!(); + println!("=== Summary ==="); + println!(" --- storage.files ---"); + println!(" scanned : {}", stats.scanned); + println!( + " already in NFC : {}", + stats.already_nfc + ); + println!( + " normalized in place (no collision) : {}", + stats.normalized_in_place + ); + println!( + " dedup-trashed (same content) : {}", + stats.deduped_same_content + ); + println!( + " renamed to .duplicate : {}", + stats.renamed_duplicate + ); + println!(" --- storage.folders ---"); + println!( + " scanned : {}", + stats.folders_scanned + ); + println!( + " already in NFC : {}", + stats.folders_already_nfc + ); + println!( + " normalized in place (no collision) : {}", + stats.folders_normalized_in_place + ); + println!( + " renamed to .duplicate : {}", + stats.folders_renamed_duplicate + ); + if dry_run { + println!(); + println!("DRY RUN — no rows were written. Re-run without --dry-run to apply."); + } + + 0 +} + +async fn load_non_trashed_files(pool: &PgPool) -> Result, Box> { + let raw = sqlx::query( + "SELECT id, folder_id, created_by, name, blob_hash, created_at + FROM storage.files + WHERE NOT is_trashed + ORDER BY created_at", + ) + .fetch_all(pool) + .await?; + + let mut out = Vec::with_capacity(raw.len()); + for r in raw { + out.push(FileRow { + id: r.try_get("id")?, + folder_id: r.try_get("folder_id")?, + created_by: r.try_get("created_by")?, + name: r.try_get("name")?, + blob_hash: r.try_get("blob_hash")?, + created_at: r.try_get("created_at")?, + }); + } + Ok(out) +} + +/// Sibling of [`load_non_trashed_files`] for folders. Ordered by +/// `created_at` so the older-wins tiebreak on collisions is +/// deterministic. Excludes trashed rows for the same reason as the +/// file scan: DB unique index is partial (`WHERE NOT is_trashed`), +/// and trashed rows will never conflict with live ones. +async fn load_non_trashed_folders( + pool: &PgPool, +) -> Result, Box> { + let raw = sqlx::query( + "SELECT id, parent_id, created_by, name, created_at + FROM storage.folders + WHERE NOT is_trashed + ORDER BY created_at", + ) + .fetch_all(pool) + .await?; + + let mut out = Vec::with_capacity(raw.len()); + for r in raw { + out.push(FolderRow { + id: r.try_get("id")?, + parent_id: r.try_get("parent_id")?, + created_by: r.try_get("created_by")?, + name: r.try_get("name")?, + created_at: r.try_get("created_at")?, + }); + } + Ok(out) +} + +/// Looks for a file in the same folder scope whose CURRENT name +/// equals `nfc_name`, excluding the row being processed. The other +/// row may itself be in non-NFC form whose normalized representation +/// happens to differ from `nfc_name`; the collision check is +/// intentionally based on stored bytes (matching the UNIQUE-index +/// semantics that this migration is repairing). +async fn find_collision( + pool: &PgPool, + row: &FileRow, + nfc_name: &str, +) -> Result, Box> { + let result = sqlx::query( + "SELECT id, folder_id, created_by, name, blob_hash, created_at + FROM storage.files + WHERE name = $1 + AND ($2::uuid IS NULL AND folder_id IS NULL + OR folder_id = $2::uuid) + AND id <> $3 + AND NOT is_trashed + LIMIT 1", + ) + .bind(nfc_name) + .bind(row.folder_id) + .bind(row.id) + .fetch_optional(pool) + .await?; + + Ok(result.map(|r| FileRow { + id: r.get("id"), + folder_id: r.get("folder_id"), + created_by: r.get("created_by"), + name: r.get("name"), + blob_hash: r.get("blob_hash"), + created_at: r.get("created_at"), + })) +} + +/// Folder-side sibling of [`find_collision`]. Same shape but keyed on +/// `parent_id` — the natural uniqueness scope for `storage.folders`. +async fn find_folder_collision( + pool: &PgPool, + row: &FolderRow, + nfc_name: &str, +) -> Result, Box> { + let result = sqlx::query( + "SELECT id, parent_id, created_by, name, created_at + FROM storage.folders + WHERE name = $1 + AND ($2::uuid IS NULL AND parent_id IS NULL + OR parent_id = $2::uuid) + AND id <> $3 + AND NOT is_trashed + LIMIT 1", + ) + .bind(nfc_name) + .bind(row.parent_id) + .bind(row.id) + .fetch_optional(pool) + .await?; + + Ok(result.map(|r| FolderRow { + id: r.get("id"), + parent_id: r.get("parent_id"), + created_by: r.get("created_by"), + name: r.get("name"), + created_at: r.get("created_at"), + })) +} + +/// Finds a free name in the form `{nfc_name}.duplicate` or +/// `{nfc_name}.duplicate-N` for `N >= 1`, scoped to the row's +/// folder. Returns the first candidate that does not currently +/// exist as a non-trashed row. +async fn find_free_duplicate_name( + pool: &PgPool, + row: &FileRow, + nfc_name: &str, +) -> Result> { + let mut suffix: u32 = 0; + loop { + let candidate = if suffix == 0 { + format!("{}.duplicate", nfc_name) + } else { + format!("{}.duplicate-{}", nfc_name, suffix) + }; + + let taken: bool = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 FROM storage.files + WHERE name = $1 + AND ($2::uuid IS NULL AND folder_id IS NULL + OR folder_id = $2::uuid) + AND id <> $3 + AND NOT is_trashed)", + ) + .bind(&candidate) + .bind(row.folder_id) + .bind(row.id) + .fetch_one(pool) + .await?; + + if !taken { + return Ok(candidate); + } + suffix = suffix.saturating_add(1); + // Safety bound — should never trigger under realistic data. + if suffix > 10_000 { + return Err(format!( + "Exhausted .duplicate-N suffixes for '{}' in scope (folder_id={:?})", + nfc_name, row.folder_id + ) + .into()); + } + } +} + +/// Folder-side sibling. Same shape as [`find_free_duplicate_name`] +/// but keyed on `parent_id`. +async fn find_free_folder_duplicate_name( + pool: &PgPool, + row: &FolderRow, + nfc_name: &str, +) -> Result> { + let mut suffix: u32 = 0; + loop { + let candidate = if suffix == 0 { + format!("{}.duplicate", nfc_name) + } else { + format!("{}.duplicate-{}", nfc_name, suffix) + }; + + let taken: bool = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 FROM storage.folders + WHERE name = $1 + AND ($2::uuid IS NULL AND parent_id IS NULL + OR parent_id = $2::uuid) + AND id <> $3 + AND NOT is_trashed)", + ) + .bind(&candidate) + .bind(row.parent_id) + .bind(row.id) + .fetch_one(pool) + .await?; + + if !taken { + return Ok(candidate); + } + suffix = suffix.saturating_add(1); + if suffix > 10_000 { + return Err(format!( + "Exhausted .duplicate-N suffixes for '{}' in scope (parent_id={:?})", + nfc_name, row.parent_id + ) + .into()); + } + } +} + +/// If the surviving (older) row's stored name is not yet in NFC, +/// UPDATE it now that the collision has been resolved. +async fn normalize_survivor_name( + pool: &PgPool, + survivor: &FileRow, + nfc_name: &str, +) -> Result<(), Box> { + if survivor.name == nfc_name { + return Ok(()); + } + sqlx::query("UPDATE storage.files SET name = $1 WHERE id = $2") + .bind(nfc_name) + .bind(survivor.id) + .execute(pool) + .await?; + Ok(()) +} + +/// Folder-side sibling of [`normalize_survivor_name`]. If the older +/// folder we kept was itself in non-NFC form, promote it to the NFC +/// name we just picked as canonical. +async fn normalize_folder_survivor_name( + pool: &PgPool, + survivor: &FolderRow, + nfc_name: &str, +) -> Result<(), Box> { + if survivor.name == nfc_name { + return Ok(()); + } + sqlx::query("UPDATE storage.folders SET name = $1 WHERE id = $2") + .bind(nfc_name) + .bind(survivor.id) + .execute(pool) + .await?; + Ok(()) +} + +/// Process every non-trashed folder, mirroring the file loop's shape. +/// Folders have no `blob_hash` so the "same content → dedup" branch is +/// absent: on collision the older folder wins its NFC name, the newer +/// gets renamed to `{nfc_name}.duplicate[-N]`. Never trashes a folder +/// — trashing would strand its subtree, and we cannot know without +/// inspection whether the newer folder was a broken second attempt +/// or an intentional sibling containing different files. Renaming is +/// the conservative choice. +async fn run_folders(pool: &PgPool, dry_run: bool, stats: &mut Stats) -> Result<(), u8> { + let rows = match load_non_trashed_folders(pool).await { + Ok(r) => r, + Err(e) => { + eprintln!("migrate nfc-filenames: folder scan failed: {e}"); + return Err(1); + } + }; + println!("Loaded {} non-trashed folder rows", rows.len()); + println!(); + + stats.folders_scanned = rows.len() as u64; + + for row in &rows { + let nfc_name = normalize_storage_name(&row.name); + if nfc_name == row.name { + stats.folders_already_nfc += 1; + continue; + } + + let collision = match find_folder_collision(pool, row, &nfc_name).await { + Ok(c) => c, + Err(e) => { + eprintln!( + "migrate nfc-filenames: folder collision query failed for {}: {e}", + row.id + ); + return Err(1); + } + }; + + match collision { + None => { + println!( + "NORMALIZE folder={} parent={:?} created_by={:?} '{}' ({}B) → '{}' ({}B)", + row.id, + row.parent_id, + row.created_by, + row.name, + row.name.len(), + nfc_name, + nfc_name.len(), + ); + if !dry_run + && let Err(e) = + sqlx::query("UPDATE storage.folders SET name = $1 WHERE id = $2") + .bind(&nfc_name) + .bind(row.id) + .execute(pool) + .await + { + eprintln!( + "migrate nfc-filenames: folder rename failed for {}: {e}", + row.id + ); + return Err(1); + } + stats.folders_normalized_in_place += 1; + } + Some(other) => { + // Older wins the canonical NFC slot; newer gets a + // `.duplicate[-N]` suffix. No dedup branch here — see + // the doc comment above. + let (older, newer) = if row.created_at <= other.created_at { + (row, &other) + } else { + (&other, row) + }; + + let disambiguated = match find_free_folder_duplicate_name(pool, newer, &nfc_name) + .await + { + Ok(n) => n, + Err(e) => { + eprintln!( + "migrate nfc-filenames: folder duplicate-name search failed for {}: {e}", + newer.id + ); + return Err(1); + } + }; + println!( + "RENAME folder-newer={} older={} created_by={:?} '{}' ({}B) → '{}' ({}B)", + newer.id, + older.id, + newer.created_by, + newer.name, + newer.name.len(), + disambiguated, + disambiguated.len(), + ); + if !dry_run { + if let Err(e) = + sqlx::query("UPDATE storage.folders SET name = $1 WHERE id = $2") + .bind(&disambiguated) + .bind(newer.id) + .execute(pool) + .await + { + eprintln!( + "migrate nfc-filenames: folder disambiguation rename failed for {}: {e}", + newer.id + ); + return Err(1); + } + if let Err(e) = normalize_folder_survivor_name(pool, older, &nfc_name).await { + eprintln!( + "migrate nfc-filenames: folder survivor rename failed for {}: {e}", + older.id + ); + return Err(1); + } + } + stats.folders_renamed_duplicate += 1; + } + } + } + Ok(()) +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs new file mode 100644 index 00000000..e8476ebe --- /dev/null +++ b/src/cli/mod.rs @@ -0,0 +1,102 @@ +//! Operator-tools subcommand tree. +//! +//! Dispatched from `src/main.rs` when the first positional arg matches +//! a known domain (`opaque`, `migrate`, `storage`). Bare `oxicloud` (or +//! oxicloud with legacy top-level flags like `--config`) falls through +//! to server startup — backwards compat with existing Docker CMD lines +//! and systemd units. +//! +//! History: this tree previously lived in a standalone `oxicloud-cli` +//! binary. Folded into the main `oxicloud` binary in v0.9.0 so the +//! release tarball ships one executable. Growth pattern preserved from +//! the old bin's header — see docs/plan/bundled-binary.md § 1b. +//! +//! ## Layout +//! +//! ```text +//! oxicloud [flags] +//! +//! Domains: +//! opaque OPAQUE aPAKE substrate management +//! setup Print a fresh ServerSetup value for +//! OXICLOUD_AUTH_OPAQUE_SERVER_SETUP +//! reset Clear envelope(s) so silent-migration +//! re-mints under current KSF +//! migrate One-time data migrations +//! nfc-filenames NFC-normalize storage.files.name +//! (pre-June-2026 databases) +//! storage Storage-config repair + crypto helpers (was --select-storage +//! and --fingerprint before v0.9.0 CLI harmonization). +//! select Set the active storage-entry backend in DB +//! fingerprint Print SSH-style fingerprint of an AES-256 key +//! ``` +//! +//! Growth pattern: each new domain gets its own module below (e.g. +//! `mod opaque`, `mod migrate`) with a `#[derive(Subcommand)]` enum +//! for its actions and a `run(action) -> u8` entrypoint. Keep +//! each module self-contained so a future extraction is a file move. +//! +//! ## Environment +//! +//! * `DATABASE_URL` — required by any subcommand that talks to the DB +//! (`opaque reset`, `migrate nfc-filenames`); not needed for pure +//! primitive helpers (`opaque setup`). Each subcommand documents its +//! own dependencies. + +use clap::{Parser, Subcommand}; + +pub mod migrate; +pub mod opaque; +pub mod storage; + +#[derive(Parser)] +#[command( + name = "oxicloud", + version, + about = "OxiCloud operator toolbox — subcommand entrypoint for \ + operational tasks that don't belong in the main server \ + binary. Run `oxicloud` (with no subcommand) to start the \ + server." +)] +struct Cli { + #[command(subcommand)] + domain: Domain, +} + +#[derive(Subcommand)] +enum Domain { + /// OPAQUE aPAKE substrate management (setup, reset). + Opaque { + #[command(subcommand)] + action: opaque::Action, + }, + /// One-time data migrations (historical schema/data fixes). + Migrate { + #[command(subcommand)] + action: migrate::Action, + }, + /// Storage-config repair + crypto helpers. + Storage { + #[command(subcommand)] + action: storage::Action, + }, +} + +/// Entrypoint called from `src/main.rs` after it detects a subcommand +/// on argv[1]. Builds a single-threaded tokio runtime — the operator +/// tools don't need multi-thread scheduling and starting a smaller +/// runtime keeps CLI invocations cheap. +pub fn run() -> u8 { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build tokio runtime for CLI"); + rt.block_on(async { + let cli = Cli::parse(); + match cli.domain { + Domain::Opaque { action } => opaque::run(action).await, + Domain::Migrate { action } => migrate::run(action).await, + Domain::Storage { action } => storage::run(action).await, + } + }) +} diff --git a/src/cli/opaque.rs b/src/cli/opaque.rs new file mode 100644 index 00000000..95c61d4a --- /dev/null +++ b/src/cli/opaque.rs @@ -0,0 +1,223 @@ +//! `opaque` subcommand domain — OPAQUE aPAKE substrate management. +//! +//! Two actions today: +//! * `setup` — mint a fresh ServerSetup for +//! `OXICLOUD_AUTH_OPAQUE_SERVER_SETUP`. Deployment-time one-off. +//! * `reset` — clear envelope columns so silent-migration re-mints them +//! under the current KSF. Used after KSF rotation. +//! +//! Previously lived in `src/bin/oxicloud-cli.rs::mod opaque` before the +//! v0.9.0 CLI/server merge — see docs/plan/bundled-binary.md § 1b. +//! Behaviour is identical; the only change is the invocation form +//! (`oxicloud opaque ` instead of `oxicloud-cli opaque `). + +use std::env; + +use clap::Subcommand; +use sqlx::{PgPool, Row}; + +use crate::infrastructure::services::opaque_service::OpaqueService; + +#[derive(Subcommand)] +pub enum Action { + /// Generate a fresh OPAQUE ServerSetup and print its base64 + /// encoding to stdout. Guidance goes to stderr so shell + /// pipelines capture cleanly. + /// + /// Run ONCE per deployment; persist the printed value as + /// `OXICLOUD_AUTH_OPAQUE_SERVER_SETUP`. Rotating this value + /// invalidates every user's OPAQUE registration — treat it + /// like your JWT secret. + Setup, + + /// Clear the OPAQUE envelope for one user or all users + /// WITHOUT touching password or setting force_password_change. + /// + /// Use case: KSF rotation. If you change + /// OXICLOUD_AUTH_OPAQUE_KSF_* values, existing envelopes + /// become cryptographically incompatible with the newly + /// published KSF — logins fail with InvalidCredentials. + /// Nulling the envelope columns forces the SPA's `/lookup` + /// to report `hasOpaque: false`, which routes the next login + /// through legacy `/api/auth/login`; silent-migration then + /// mints a fresh envelope under the CURRENT KSF. Passwords + /// are unchanged. + /// + /// NOT for forgotten-passphrase recovery — use the admin + /// password-reset endpoint (`PUT /api/admin/users/{id}/password`) + /// which sets a temp password + force_change flag in one shot. + Reset { + /// Email OR username to reset (dispatched on `@` presence, + /// same rule as `POST /api/auth/login`). + #[arg(long, conflicts_with = "all")] + user: Option, + + /// Reset every user with an OPAQUE envelope. + #[arg(long, conflicts_with = "user")] + all: bool, + + /// Print what would change without touching the DB. + #[arg(long)] + dry_run: bool, + }, +} + +pub async fn run(action: Action) -> u8 { + match action { + Action::Setup => run_setup(), + Action::Reset { user, all, dry_run } => run_reset(user, all, dry_run).await, + } +} + +fn run_setup() -> u8 { + // Match the legacy `opaque-setup` bin's contract: + // - value on stdout, no trailing commentary (pipeline-safe) + // - guidance on stderr + let b64 = OpaqueService::generate_server_setup_b64(); + println!("{b64}"); + eprintln!(); + eprintln!("=== OPAQUE server setup generated. ==="); + eprintln!("Persist the line above in OXICLOUD_AUTH_OPAQUE_SERVER_SETUP."); + eprintln!("NEVER rotate: rotating invalidates every user's registration."); + eprintln!("Treat this value like your JWT secret."); + 0 +} + +async fn run_reset(user: Option, all: bool, dry_run: bool) -> u8 { + // clap enforces `conflicts_with`, but not "at least one of". + // Belt-and-braces check here so the failure is explicit. + if user.is_none() && !all { + eprintln!("opaque reset: pass either --user or --all"); + return 2; + } + + let database_url = match env::var("DATABASE_URL") { + Ok(v) => v, + Err(_) => { + eprintln!("opaque reset: DATABASE_URL not set"); + return 2; + } + }; + let pool = match PgPool::connect(&database_url).await { + Ok(p) => p, + Err(e) => { + eprintln!("opaque reset: failed to connect to database: {e}"); + return 1; + } + }; + + // Preview the affected row set before writing. Doubles as + // dry-run output and as diagnostics when --user matches nothing. + // Envelope-presence bool lets the operator see which rows had + // an envelope vs which only carry a stale migration mark. + let select_sql = if all { + r#" + SELECT id, email, (opaque_envelope IS NOT NULL) AS had_envelope + FROM auth.users + WHERE opaque_envelope IS NOT NULL + OR opaque_migrated_at IS NOT NULL + ORDER BY email + "# + } else { + r#" + SELECT id, email, (opaque_envelope IS NOT NULL) AS had_envelope + FROM auth.users + WHERE CASE WHEN $1 LIKE '%@%' THEN email = $1 ELSE username = $1 END + "# + }; + let rows_result = if all { + sqlx::query(select_sql).fetch_all(&pool).await + } else { + let ident = user.as_deref().unwrap(); + sqlx::query(select_sql).bind(ident).fetch_all(&pool).await + }; + let rows = match rows_result { + Ok(r) => r, + Err(e) => { + eprintln!("opaque reset: query failed: {e}"); + return 1; + } + }; + if rows.is_empty() { + if all { + println!("opaque reset: no users have an OPAQUE envelope — nothing to do."); + return 0; + } else { + eprintln!( + "opaque reset: no user matches --user {} — nothing changed.", + user.as_deref().unwrap_or("") + ); + return 1; + } + } + + println!( + "opaque reset ({}): {} row(s) to affect", + if dry_run { + "DRY RUN — no writes" + } else { + "EXECUTING" + }, + rows.len() + ); + for row in &rows { + let id: uuid::Uuid = row.get("id"); + let email: String = row.get("email"); + let had_envelope: bool = row.get("had_envelope"); + println!( + " {} {} {}", + id, + email, + if had_envelope { + "had-envelope" + } else { + "no-envelope-had-migrated-mark" + } + ); + } + if dry_run { + return 0; + } + + // Actual UPDATE. Kept identical in shape to the SELECT above so + // the planner sees the same query pattern for both. We + // DELIBERATELY do NOT touch password_hash or + // force_password_change_at_next_login — this tool is scoped + // to "the passwords are fine, the envelopes are stale." + let update_sql_all = r#" + UPDATE auth.users + SET opaque_envelope = NULL, + opaque_ciphersuite_version = NULL, + opaque_registered_at = NULL, + opaque_migrated_at = NULL + WHERE opaque_envelope IS NOT NULL + OR opaque_migrated_at IS NOT NULL + "#; + let update_sql_one = r#" + UPDATE auth.users + SET opaque_envelope = NULL, + opaque_ciphersuite_version = NULL, + opaque_registered_at = NULL, + opaque_migrated_at = NULL + WHERE CASE WHEN $1 LIKE '%@%' THEN email = $1 ELSE username = $1 END + "#; + let write_result = if all { + sqlx::query(update_sql_all).execute(&pool).await + } else { + let ident = user.as_deref().unwrap(); + sqlx::query(update_sql_one).bind(ident).execute(&pool).await + }; + let affected = match write_result { + Ok(r) => r.rows_affected(), + Err(e) => { + eprintln!("opaque reset: update failed: {e}"); + return 1; + } + }; + println!( + "opaque reset: cleared envelope columns on {affected} row(s). \ + Users log in with their existing password; silent-migration \ + re-mints envelopes under the current KSF on next login." + ); + 0 +} diff --git a/src/cli/storage.rs b/src/cli/storage.rs new file mode 100644 index 00000000..ff3d1b01 --- /dev/null +++ b/src/cli/storage.rs @@ -0,0 +1,157 @@ +//! `storage` subcommand domain — storage-config repair + crypto helpers. +//! +//! Two actions today: +//! * `select ` — set `admin_settings.storage.active_backend_name` +//! in the DB to the named entry and exit. Used to unblock boot after +//! renaming or removing a storage entry in `.env` while the DB still +//! points at the old name (the server aborts boot with a pointer to +//! this subcommand when that happens). See +//! `docs/plan/storage-multi-entry.md` § Fallback. +//! * `fingerprint ` — print the SSH-style colon-hex +//! fingerprint of a base64-encoded AES-256 key. Matches the +//! `head_key_fp` field the `backend_rotate` job reports on completion +//! and the raw `` field embedded in every v1 blob header — so +//! an admin can pair a key in `OXICLOUD_STORAGE__ENCRYPTION_KEY` +//! with the current on-disk head and safely drop any key whose +//! fingerprint does NOT match the last-successful rotate. +//! +//! Both actions previously lived as top-level flags (`--select-storage`, +//! `--fingerprint`) on the `oxicloud` binary. Moved into the subcommand +//! tree in v0.9.0 for CLI consistency — see docs/plan/bundled-binary.md +//! § 1c. Behaviour is identical. + +use std::env; +use std::io::Read; + +use clap::Subcommand; + +use crate::common::config::{AppConfig, fingerprint_from_base64_key}; +use crate::infrastructure::services::entry_backend::persist_active_backend_name; + +#[derive(Subcommand)] +pub enum Action { + /// Select the active storage-entry backend. Writes + /// `admin_settings.storage.active_backend_name = ` in the DB + /// and exits. Does NOT boot the server. Use to recover from the + /// "boot fails on missing entry" case after renaming or removing a + /// storage entry in `.env`. + /// + /// The named entry MUST appear in `OXICLOUD_STORAGE_ENTRIES` — this + /// subcommand re-parses the same env the server would parse at boot, + /// so a successful run guarantees the subsequent boot will find the + /// entry (no drift between the two code paths). + Select { + /// Storage-entry name (must appear in OXICLOUD_STORAGE_ENTRIES). + name: String, + }, + + /// Print the SSH-style colon-hex fingerprint (16-hex, 8-byte + /// truncation of sha256) of a base64-encoded AES-256 key. + /// + /// Matches the `head_key_fp` field the `backend_rotate` job reports + /// on completion, and the raw `` field embedded in every v1 + /// blob header. Used to identify which key in + /// `OXICLOUD_STORAGE__ENCRYPTION_KEY` corresponds to the current + /// on-disk head — safe to drop any key whose fingerprint does NOT + /// match the last-successful rotate's `head_key_fp`. + /// + /// Pass `-` to read the key from stdin so it never touches shell + /// history: + /// + /// ```text + /// echo -n '' | oxicloud storage fingerprint - + /// ``` + Fingerprint { + /// Base64-encoded AES-256 key, or `-` to read the key from stdin. + key: String, + }, +} + +pub async fn run(action: Action) -> u8 { + match action { + Action::Select { name } => run_select(&name).await, + Action::Fingerprint { key } => run_fingerprint(&key), + } +} + +/// Verify `name` is declared in the current env, UPDATE +/// `admin_settings.storage.active_backend_name`, exit. +/// +/// Loading AppConfig here re-runs the same env-parse the server does +/// at boot, so a successful `storage select` guarantees a subsequent +/// normal boot will find the entry — no drift between the two code +/// paths. +async fn run_select(name: &str) -> u8 { + let config = AppConfig::from_env(); + if config.storage_entries.is_empty() { + eprintln!( + "OXICLOUD_STORAGE_ENTRIES is not set (or synthesised — legacy path). \ + `storage select` needs at least one named entry to switch to." + ); + return 2; + } + if !config.storage_entries.iter().any(|e| e.name == name) { + let available = config + .storage_entries + .iter() + .map(|e| e.name.as_str()) + .collect::>() + .join(", "); + eprintln!( + "entry `{name}` is not declared in OXICLOUD_STORAGE_ENTRIES. \ + Available: [{available}]" + ); + return 2; + } + + let db_url = match env::var("DATABASE_URL") { + Ok(v) => v, + Err(_) => { + eprintln!( + "DATABASE_URL not set — `storage select` needs the same DB the server \ + would boot on" + ); + return 2; + } + }; + let pool = match sqlx::PgPool::connect(&db_url).await { + Ok(p) => p, + Err(e) => { + eprintln!("failed to connect to DATABASE_URL: {e}"); + return 1; + } + }; + + if let Err(e) = persist_active_backend_name(&pool, name).await { + eprintln!("failed to write admin_settings.storage.active_backend_name = `{name}`: {e}"); + return 1; + } + + println!( + "active_backend_name = `{name}` written to admin_settings. Restart the server to switch." + ); + 0 +} + +fn run_fingerprint(key: &str) -> u8 { + let key_b64 = if key == "-" { + let mut buf = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut buf) { + eprintln!("failed to read key from stdin: {e}"); + return 2; + } + buf.trim().to_string() + } else { + key.to_string() + }; + match fingerprint_from_base64_key(&key_b64) { + Ok(fp) => { + println!("{fp}"); + 0 + } + Err(e) => { + eprintln!("storage fingerprint: {e}"); + 2 + } + } +} diff --git a/src/common/config.rs b/src/common/config.rs index fc3b54bb..67001e03 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -2,6 +2,8 @@ use std::env; use std::path::PathBuf; use std::time::Duration; +use crate::infrastructure::scheduler::JobRunArgs; + /// Cache configuration #[derive(Debug, Clone)] pub struct CacheConfig { @@ -512,7 +514,7 @@ impl KeyPair { /// truncation as the v1 header's `` field and the /// `head_key_fp` reported by `backend_rotate` on completion, so /// operators can cross-reference the boot log against a rotate - /// report or the CLI's `oxicloud --fingerprint ` + /// report or the CLI's `oxicloud storage fingerprint ` /// output without any format conversion. /// /// Returns `None` for `CipherKind::None` (nothing to @@ -723,7 +725,7 @@ pub fn parse_encryption_pair_list(entry_name: &str, raw: &str) -> Result` CLI subcommand so +/// Used by the `oxicloud storage fingerprint ` CLI subcommand so /// admins can identify which key in their `.env` corresponds to the /// `head_key_fp` a `backend_rotate` run reported on completion — /// see `docs/plan/storage-key-rotation.md`. @@ -2280,6 +2282,155 @@ pub struct GrantCleanupConfig { pub interval_hours: u64, } +/// One job to dispatch once at startup, parsed from an entry of +/// `OXICLOUD_STARTUP_JOBS`. +/// +/// **Why this exists.** Scheduled ticks deliberately never pass +/// `repair` — a job that deletes on its default setting is the thing +/// no-silent-auto-repair forbids. But that leaves the migration jobs in +/// a state where an operator who never opens the admin panel imports +/// forever and never drains: the sidecars are fully redundant, and +/// nothing removes them. Naming the job in configuration IS the +/// deliberate operator action; it just gets taken once, at boot, +/// instead of every time. +/// +/// Not a general "run everything in repair mode" switch. Each job is +/// named individually, and the flags are per job. +/// Holds a [`JobRunArgs`] rather than re-listing its fields. They are +/// the same four flags with the same meanings, and a copy here would +/// have to be found and updated the next time the scheduler grows a +/// fifth — silently ignoring it in configuration until someone noticed. +#[derive(Debug, Clone, Default)] +pub struct StartupJob { + /// Registered job name — must match `JobHandler::name`. + pub name: String, + /// Forwarded verbatim to `JobRegistry::trigger`. + pub args: JobRunArgs, +} + +/// Parse one `OXICLOUD_STARTUP_JOBS` entry: `name`, or +/// `name?repair=true&deep=true`. +/// +/// The query syntax is the one an operator already types at +/// `POST /api/admin/jobs/{name}/trigger?repair=true`, so the value is +/// literally the request they would otherwise make by hand. +/// +/// **Errors on anything it does not recognise**, rather than ignoring +/// it. A silently-dropped `?repare=true` typo would leave the job +/// running in discovery-only mode forever while the operator believed +/// the tier was draining — the failure would surface as "the migration +/// never finishes" months later, with nothing in the logs pointing at +/// the config. Same reasoning as fail-fast on any broken config. +fn parse_startup_job(raw: &str) -> Result { + let raw = raw.trim(); + let (name, query) = match raw.split_once('?') { + Some((n, q)) => (n.trim(), q), + None => (raw, ""), + }; + if name.is_empty() { + return Err("empty job name".to_string()); + } + + let mut job = StartupJob { + name: name.to_string(), + args: JobRunArgs::default(), + }; + + for pair in query.split('&').filter(|p| !p.is_empty()) { + let (key, value) = pair + .split_once('=') + .ok_or_else(|| format!("`{pair}` is not key=value (job `{name}`)"))?; + // Booleans accept only `true`/`false` — the same rule the HTTP + // trigger enforces, so a value that works in one place works in + // the other. See memory `bug_axum_query_bool_only_accepts_true_false`. + let as_bool = || match value { + "true" => Ok(true), + "false" => Ok(false), + other => Err(format!( + "`{key}={other}` on job `{name}`: expected true or false" + )), + }; + match key { + "force" => job.args.force = as_bool()?, + "deep" => job.args.deep = as_bool()?, + "repair" => job.args.repair = as_bool()?, + "storage" => job.args.storage = Some(value.to_string()), + other => { + return Err(format!( + "unknown flag `{other}` on job `{name}`: expected force, deep, repair \ + or storage" + )); + } + } + } + Ok(job) +} + +/// What runs at boot when `OXICLOUD_STARTUP_JOBS` is unset. +/// +/// **Both migration jobs, both in repair mode** — they import their +/// sidecars and then delete them. Chosen deliberately: an operator who +/// never edits `.env` is the normal case, and a migration nobody +/// triggers never finishes, so a default that only imports would leave +/// every untouched deployment carrying a fully-redundant `.thumbnails/` +/// forever. +/// +/// This is a destructive default, which is a real exception to +/// no-silent-auto-repair, so what makes it safe has to hold: +/// +/// * **Nothing is deleted before its replacement has been read back.** +/// `verify_and_unlink` imports, reads the blob back through the normal +/// stack, and only then unlinks. A store that reported success but +/// landed unreadable keeps its sidecar. That readback is the whole +/// safety argument — it matters most for `thumb_attached_import`, +/// whose bytes are user-uploaded previews with no render path, so a +/// wrong deletion there is permanent where a wrong deletion of a +/// server-rendered thumbnail costs only a re-render. +/// * **Sidecars whose source is gone are deleted without a readback**, +/// because there is nothing to read back and nothing can ever +/// reference them again. Unrecoverable and unreachable are different +/// things; these are both. +/// * **Every deletion is audited**, so an operator can reconstruct what +/// a boot removed and from which source. +/// +/// The consequence to be aware of when changing this: an upgrade +/// deletes on first boot, in every deployment at once, with no operator +/// action. A regression in the readback path would therefore be +/// simultaneous and unrecoverable. Treat that code as load-bearing. +/// +/// Set `OXICLOUD_STARTUP_JOBS=` (empty) to disable startup jobs +/// entirely; any explicit value replaces this list rather than adding +/// to it. +/// `transcode_import` joins them for the same reason and on the same +/// terms. Its artifacts are the most disposable of the three — a +/// transcode is a pure function of its source, so anything deleted in +/// error is recomputed on the next request — and its `.skip` markers +/// collapse to one row per distinct content, which is the saving that +/// only happens once the import runs. +const DEFAULT_STARTUP_JOBS: &str = "thumb_derived_import?repair=true,\ + thumb_attached_import?repair=true,\ + transcode_import?repair=true"; + +/// Parse the whole `OXICLOUD_STARTUP_JOBS` value. Empty → no startup +/// jobs (an explicit opt-out); unset → [`DEFAULT_STARTUP_JOBS`]. +/// +/// # Panics +/// +/// On any malformed entry. A startup-job list that half-parses is worse +/// than one that fails: the server would come up looking healthy with a +/// migration that never runs. +fn parse_startup_jobs(raw: &str) -> Vec { + raw.split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|entry| { + parse_startup_job(entry).unwrap_or_else(|e| { + panic!("OXICLOUD_STARTUP_JOBS: {e}"); + }) + }) + .collect() +} + impl Default for GrantCleanupConfig { fn default() -> Self { Self { @@ -2543,6 +2694,17 @@ pub struct AppConfig { /// bind to loopback / a private interface without exposing /// metrics publicly. pub metrics_listen: Option, + /// Jobs to dispatch once, in the background, after the scheduler is + /// ready. Env: `OXICLOUD_STARTUP_JOBS` — comma-separated, each entry + /// `name` or `name?repair=true`, mirroring the admin trigger URL. + /// + /// Empty by default. Intended for the migration jobs, whose + /// scheduled ticks import but deliberately never delete: naming one + /// here is the operator's standing consent to the deletion, given + /// once in configuration instead of per run in the panel. + /// + /// Dispatch is non-blocking — readiness never waits on a job. + pub startup_jobs: Vec, /// Cache configuration pub cache: CacheConfig, /// Timeout configuration @@ -2671,6 +2833,7 @@ impl Default for AppConfig { plugins: PluginConfig::default(), faces: FacesConfig::default(), metrics_listen: None, + startup_jobs: parse_startup_jobs(DEFAULT_STARTUP_JOBS), } } } @@ -2718,6 +2881,19 @@ impl AppConfig { } } + // Jobs to fire once at boot. Unset keeps DEFAULT_STARTUP_JOBS (set + // by `Default`); any explicit value REPLACES it, and an empty value + // is the opt-out. + // + // Panics on a malformed entry rather than warning: unlike metrics, + // a startup job that silently fails to parse leaves a migration + // that never runs, and the symptom ("the tier never drained") + // surfaces months later with nothing pointing back at the config + // line. + if let Ok(raw) = env::var("OXICLOUD_STARTUP_JOBS") { + config.startup_jobs = parse_startup_jobs(&raw); + } + // Database configuration if let Ok(connection_string) = env::var("OXICLOUD_DB_CONNECTION_STRING") { config.database.connection_string = connection_string; @@ -3783,6 +3959,96 @@ pub fn default_config() -> AppConfig { mod tests { use super::*; + #[test] + fn startup_job_parses_name_and_flags() { + let jobs = parse_startup_jobs( + "thumb_derived_import?repair=true, thumb_attached_import ,blobs_consistency?deep=true&force=false", + ); + assert_eq!(jobs.len(), 3); + + assert_eq!(jobs[0].name, "thumb_derived_import"); + assert!(jobs[0].args.repair); + assert!(!jobs[0].args.deep); + + // Bare name → all flags default off, which is the discovery-only + // run. Naming a migration job without `repair` imports and stops. + assert_eq!(jobs[1].name, "thumb_attached_import"); + assert!(!jobs[1].args.repair); + + assert!(jobs[2].args.deep); + assert!(!jobs[2].args.force); + } + + #[test] + fn startup_job_accepts_storage_scope() { + let jobs = parse_startup_jobs("backend_consistency?storage=s3_prod&deep=true"); + assert_eq!(jobs[0].args.storage.as_deref(), Some("s3_prod")); + assert!(jobs[0].args.deep); + } + + #[test] + fn startup_jobs_empty_value_is_the_opt_out() { + assert!(parse_startup_jobs("").is_empty()); + assert!(parse_startup_jobs(" , ,").is_empty()); + } + + /// Both migration jobs drain themselves out of the box, deletion + /// included. Pinned rather than left implicit because this is a + /// destructive default: it deletes on first boot after an upgrade, + /// everywhere, with no operator action. Whoever changes this line + /// should have to change a test that says so. + /// + /// What keeps it safe is the readback in `verify_and_unlink` — import, + /// read the blob back through the normal stack, and only then unlink. + /// That matters most for `thumb_attached_import`, whose bytes are + /// user-uploaded and have no render path to rebuild them. + #[test] + fn default_startup_jobs_drain_both_thumbnail_tiers() { + let jobs = AppConfig::default().startup_jobs; + let names: Vec<&str> = jobs.iter().map(|j| j.name.as_str()).collect(); + assert_eq!( + names, + [ + "thumb_derived_import", + "thumb_attached_import", + "transcode_import" + ] + ); + assert!(jobs.iter().all(|j| j.args.repair)); + assert!(jobs.iter().all(|j| !j.args.deep && !j.args.force)); + } + + /// A misspelled flag must not parse. Silently ignoring `repare=true` + /// leaves the job in discovery-only mode while the operator believes + /// the tier is draining — a failure that surfaces months later as + /// "the migration never finished", with nothing pointing at the + /// config line. + #[test] + #[should_panic(expected = "unknown flag `repare`")] + fn startup_job_rejects_a_misspelled_flag() { + parse_startup_jobs("thumb_derived_import?repare=true"); + } + + /// Booleans take only true/false — the same rule the HTTP trigger + /// enforces, so a value that works in one place works in the other. + #[test] + #[should_panic(expected = "expected true or false")] + fn startup_job_rejects_a_non_boolean_flag_value() { + parse_startup_jobs("thumb_derived_import?repair=yes"); + } + + #[test] + #[should_panic(expected = "not key=value")] + fn startup_job_rejects_a_valueless_flag() { + parse_startup_jobs("thumb_derived_import?repair"); + } + + #[test] + #[should_panic(expected = "empty job name")] + fn startup_job_rejects_flags_with_no_job() { + parse_startup_jobs("?repair=true"); + } + #[test] fn empty_allowlist_accepts_any_email() { let cfg = MagicLinkConfig::default(); @@ -4330,7 +4596,7 @@ mod tests { // SSH-style 8-byte colon-hex (16 hex + 7 colons = 23 chars) // so operators can cross-reference against the v1 header's // `` field + `backend_rotate`'s `head_key_fp` - // output + the `oxicloud --fingerprint` CLI. + // output + the `oxicloud storage fingerprint` CLI. let pairs = parse_encryption_pair_list("t", &format!("aes-256-gcm:{K1_B64},none:")).unwrap(); let fp0 = pairs[0].fingerprint_short().unwrap(); diff --git a/src/common/di.rs b/src/common/di.rs index 5a6fb9f8..8daa9324 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -313,7 +313,7 @@ impl AppServiceFactory { tracing::info!( "Storage: no active_backend_name set in DB — defaulting to first entry \ `{}` (declared first in OXICLOUD_STORAGE_ENTRIES). Set explicitly via \ - the admin storage tab or `oxicloud --select-storage ` to pin.", + the admin storage tab or `oxicloud storage select ` to pin.", first.name, ); first @@ -451,6 +451,17 @@ impl AppServiceFactory { ); dedup_service.initialize().await?; + // Hand the transcode service its derived tier. + // + // Deferred rather than injected at construction because that happens + // ~240 lines above this, before `DedupService` exists, and the + // retrieval path that needs the transcode service is wired earlier + // still. Reordering DI to make the dependency a constructor argument + // would move more than it is worth; the service treats a missing + // handle as "local cache only", which is exactly its pre-derived-tier + // behaviour. + image_transcode_service.attach_dedup(dedup_service.clone()); + // One-time background migration: re-chunk pre-CDC whole-file blobs // into chunk manifests so Range reads (and, with encryption, partial // decrypts) stop paying for the entire blob. No-op once converged. @@ -1460,6 +1471,85 @@ impl AppServiceFactory { .register_recoverable_job(&core.job_registry, &job_store_provider_dyn) .await; + // Reconciles `chunk_manifests.ref_count` — the SECOND reference + // counter, and the one nothing verified before. add_reference bumps + // it first and only falls back to storage.blobs.ref_count, so every + // CDC file (and every derived artifact, once those land) counts here + // rather than at the chunk level. Uses the same registry dedup_gc + // reaps from, so the two cannot disagree. + // See docs/plan/derived-blobs.md. + let _ = Arc::new( + crate::infrastructure::services::manifests_consistency_service::ManifestsConsistencyCheck::new( + maintenance_pool.clone(), + core.dedup_service.reference_registry(), + ), + ) + .register_recoverable_job(&core.job_registry, &job_store_provider_dyn) + .await; + + // Step 10 migration tenant: backfills `content_derived_blobs` from + // the on-disk thumbnail sidecars that predate it. Idempotent, so it + // is safe to trigger repeatedly — Phase 3 (deleting the sidecars) is + // gated on a run reporting zero imported. Registered unconditionally + // rather than behind a flag: a migration nobody can find is a + // migration nobody runs. + // + // `.thumbnails` lives under the storage path, matching + // `ThumbnailService::new(&self.storage_path, …)` above. + let _ = Arc::new( + crate::infrastructure::services::thumb_derived_import_service::ThumbDerivedImport::new( + std::path::Path::new(&self.storage_path).join(".thumbnails"), + core.dedup_service.clone(), + ), + ) + .register_recoverable_job(&core.job_registry, &job_store_provider_dyn) + .await; + + // Step 7 migration tenant: drains `.transcoded/` the same way the + // thumbnail imports drain `.thumbnails/`, with one extra step — + // the legacy tree is keyed by FILE (`{file_id}.webp`) while the + // destination is keyed by CONTENT, so every entry is re-keyed + // through `storage.files` on the way in. Entries naming the same + // content collapse into one row, which is the saving this migration + // exists for; a sandbox with five `.skip` markers had three of them + // describing one image. + let _ = Arc::new( + crate::infrastructure::services::transcode_import_service::TranscodeImport::new( + std::path::Path::new(&self.storage_path).join(".transcoded"), + core.dedup_service.clone(), + maintenance_pool.clone(), + ), + ) + .register_recoverable_job(&core.job_registry, &job_store_provider_dyn) + .await; + + // Both satellite tables, checked for mappings whose Blob is gone. + // Nothing else can: a row whose SOURCE was reaped still holds a valid + // reference to a real artifact with a correct refcount, so every + // other check agrees the system is healthy while the artifact is + // pinned forever. Read-only. + let _ = Arc::new( + crate::infrastructure::services::satellites_consistency_service::SatellitesConsistencyCheck::new( + maintenance_pool.clone(), + ), + ) + .register_recoverable_job(&core.job_registry, &job_store_provider_dyn) + .await; + + // Its file-keyed twin: `ext-{file_id}.jpg` previews the user uploaded, + // which no copy path duplicates today. Separate job, separate keying — + // routing these into the content-keyed table would share one user's + // preview onto every file with identical content. + let _ = Arc::new( + crate::infrastructure::services::thumb_attached_import_service::ThumbAttachedImport::new( + std::path::Path::new(&self.storage_path).join(".thumbnails"), + core.dedup_service.clone(), + maintenance_pool.clone(), + ), + ) + .register_recoverable_job(&core.job_registry, &job_store_provider_dyn) + .await; + // Third recoverable-run tenant. Iterates `storage.files` // and reports parent-folder-trashed cascade misses, // `missing_blob` (data-loss indicator — file references @@ -1475,24 +1565,24 @@ impl AppServiceFactory { .register_recoverable_job(&core.job_registry, &job_store_provider_dyn) .await; - // Fourth recoverable-run tenant. Iterates `storage.blobs` - // and verifies each row against the physical backend AND - // against the reference-counting invariants that `dedup_gc` - // relies on. Three per-row checks (subject-iteration in - // action): `blob_missing_from_backend` (data_loss, bytes - // gone from disk), `refcount_mismatch` (inconsistent, - // dedup counter drift), and `blob_corrupted` (data_loss, - // deep mode only — bit-rot). Complements - // `files_consistency` without doubling work: probing - // per-unique-blob preserves dedup savings vs probing - // per-file-chunk. See memory - // `project_cdc_dual_storage_registries` for the rationale. + // Fourth recoverable-run tenant. Iterates `storage.blobs` and + // checks the reference-counting invariant `dedup_gc` relies on: + // `refcount_mismatch` (inconsistent — an under-count lets GC reap + // a live blob, an over-count pins a dead one), repairable under + // `?repair=true`. + // + // DB-only, and takes no backend. Physical checks — missing bytes, + // orphaned bytes, bit-rot — all belong to `backend_consistency`, + // which merge-joins the backend enumeration against this same + // table in one pass. This tenant used to probe the backend once + // per row for missing bytes, which found strictly less than the + // merge-join at N round-trips instead of one enumeration. let _ = Arc::new( crate::infrastructure::services::blobs_consistency_service::BlobsConsistencyCheck::new( maintenance_pool.clone(), - core.blob_backend.clone(), - core.config.storage_entries.clone(), - self.storage_path.clone(), + // Same registry instance GC reaps from — see + // DedupService::reference_registry. + core.dedup_service.reference_registry(), ), ) .register_recoverable_job(&core.job_registry, &job_store_provider_dyn) @@ -2126,6 +2216,38 @@ impl AppServiceFactory { let mut core = core; core.zip_service = Some(zip_service); + // Session liveness tracker — spawns its own 30 s flush + // loop at construction. Only built when auth (and thus + // sessions) exist; when auth is off this is `None` and + // the middleware never calls it. Uses the maintenance + // pool so background flushes don't compete with + // request-serving connections. See + // [`LastSeenTracker`](crate::infrastructure::services::last_seen_tracker). + let last_seen_tracker = if auth_services.is_some() { + Some( + crate::infrastructure::services::last_seen_tracker::LastSeenTracker::start( + maintenance_pool.clone(), + ), + ) + } else { + None + }; + + // Session-liveness Prometheus poller — three COUNT(*) reads + // every 30 s, publishing `oxicloud_sessions_active`, + // `_active_users`, `_total_non_revoked`. Only spawned when + // the Prometheus recorder is installed (i.e., + // `OXICLOUD_METRICS_LISTEN` is set) — without the recorder + // the `metrics::gauge!(...)` calls are no-ops and the + // periodic PG hits would be pure waste. Requires auth for + // the same reason as `last_seen_tracker`: no sessions to + // count without it. + if auth_services.is_some() && self.config.metrics_listen.is_some() { + crate::infrastructure::services::session_liveness_gauges::spawn( + maintenance_pool.clone(), + ); + } + // 9. Assemble final AppState let mut app_state = AppState { core, @@ -2158,6 +2280,7 @@ impl AppServiceFactory { people_service, storage_usage_service, grant_cleanup_service, + last_seen_tracker, calendar_service: None, calendar_use_case: None, addressbook_use_case: None, @@ -2741,6 +2864,116 @@ impl AppServiceFactory { registered ); + // `OXICLOUD_STARTUP_JOBS` — dispatch each named job once, now. + // + // Exists for the migration jobs. Their scheduled ticks import but + // never delete (`repair` defaults false, per no-silent-auto-repair), + // so a deployment whose operator never opens the admin panel keeps + // importing sidecars it already imported and never drains the + // directory. Naming the job in configuration IS the deliberate + // consent that rule asks for; it is simply given once, at boot, + // rather than per run. + // + // Validated here, dispatched in the background: + // + // * Unknown names **panic**. The registry is fully populated at this + // point, so a name that does not resolve is a typo or a rename, and + // the failure mode of ignoring it is a migration that silently + // never runs. Fail at boot, where the operator is watching. + // * Dispatch is `tokio::spawn` — readiness must never wait on a job + // that walks a filesystem for hours. + // * Sequential within the task, not concurrent: these jobs contend + // for the same directory and DB, and the exclusivity gate would + // turn overlap into a skipped run rather than a queued one. + // * Safe on every boot, including a crash loop: each is idempotent + // and resumable, and once drained a run is a `read_dir` that + // returns nothing. + // + // **Killed mid-run, this resumes from the cursor.** The boot + // recovery sweep runs earlier in this function and flips every row + // the dead process abandoned in `Running` to `Paused`, keeping its + // cursor. `run_or_resume` then picks Resume over a fresh start, so + // a job interrupted by a restart continues where it stopped rather + // than rescanning from the beginning — and a long migration + // completes across however many restarts it takes. + // + // That is a deliberate exception to `boot_recovery_sweep`'s "we do + // not auto-resume; operators trigger the resume explicitly". The + // rule exists so a restart never silently resumes work nobody + // asked for. Here somebody did ask, in configuration, and the whole + // point of the option is not having to ask again. The exception is + // scoped to the named jobs; every other paused run still waits for + // an operator. + // + // The resumed run keeps the flags it started with — `repair` and + // `deep` are persisted to the run's `params` on the fresh open and + // read back on resume — so editing the config mid-migration does + // not retroactively change a run already in flight. + if !self.config.startup_jobs.is_empty() { + let mut planned = Vec::with_capacity(self.config.startup_jobs.len()); + for job in &self.config.startup_jobs { + if app_state.core.job_registry.get(&job.name).await.is_none() { + panic!( + "OXICLOUD_STARTUP_JOBS names `{}`, which is not a registered job. \ + Check the spelling against GET /api/admin/jobs.", + job.name + ); + } + planned.push(job.clone()); + } + + let registry = app_state.core.job_registry.clone(); + tokio::spawn(async move { + for job in planned { + // Audited, not merely logged: a startup job may delete + // files, and "who asked for this" must be answerable + // afterwards. The answer is the configuration, which is + // exactly what this line records. + tracing::info!( + target: "audit", + event = "job.startup_trigger", + job = %job.name, + force = job.args.force, + deep = job.args.deep, + repair = job.args.repair, + storage = ?job.args.storage, + "👮🏻‍♂️ dispatching `{}` from OXICLOUD_STARTUP_JOBS", + job.name, + ); + match registry.trigger(&job.name, &job.args).await { + // Debug, not info. The engine already logs every + // dispatch as `job.run` with the outcome and timing — + // that is the point of routing through `trigger` + // rather than calling handlers directly. An info line + // here made every startup job report completion + // twice, from two layers, saying the same thing. The + // `job.startup_trigger` audit line above already + // records that the startup path was the caller. + Some(outcome) => tracing::debug!( + target: "oxicloud::scheduler", + event = "job.startup_completed", + job = %job.name, + outcome = outcome.kind(), + "startup job `{}` finished ({})", + job.name, + outcome.kind(), + ), + // Unreachable — the name was resolved above, and + // nothing unregisters. Logged rather than panicking + // because this is a detached task by then. + None => tracing::error!( + target: "oxicloud::scheduler", + event = "job.startup_vanished", + job = %job.name, + "startup job `{}` disappeared from the registry between \ + validation and dispatch", + job.name, + ), + } + } + }); + } + Ok(app_state) } } @@ -2952,6 +3185,16 @@ pub struct AppState { pub grant_cleanup_service: Option< Arc, >, + /// Per-session liveness tracker — the auth middleware calls + /// `stamp(session_id)` after every successful token validation, + /// and a background loop flushes the DashMap to `auth.sessions. + /// last_seen_at` every 30 s (batched UNNEST UPDATE). `None` + /// when auth is disabled — nothing to track. See + /// [`LastSeenTracker`](crate::infrastructure::services::last_seen_tracker) + /// for the contract and `docs/plan/sessions.md` for the design. + pub last_seen_tracker: Option< + Arc, + >, pub calendar_service: Option>, pub calendar_use_case: Option>, pub addressbook_use_case: Option>, diff --git a/src/common/stubs.rs b/src/common/stubs.rs index 952874f7..63a0ac32 100644 --- a/src/common/stubs.rs +++ b/src/common/stubs.rs @@ -756,6 +756,15 @@ impl DedupPort for StubDedupPort { false } + async fn find_derived_blob( + &self, + _source_hash: &str, + _kind: &str, + _variant: &str, + ) -> Option { + None + } + async fn get_blob_metadata(&self, _hash: &str) -> Option { None } diff --git a/src/domain/entities/calendar_event.rs b/src/domain/entities/calendar_event.rs index 2fb6c4e9..fbbc0b83 100644 --- a/src/domain/entities/calendar_event.rs +++ b/src/domain/entities/calendar_event.rs @@ -1105,9 +1105,15 @@ impl CalendarEvent { } // Standard UTC form: YYYYMMDDTHHMMSSZ, 16 chars, trailing 'Z'. - // Floating-time (no 'Z') and TZID-anchored forms aren't yet - // supported — future work when we tackle VTIMEZONE properly. - if value.len() < 15 || !value.ends_with('Z') { + // Floating-time (no 'Z', RFC 5545 §3.3.5) is what calendar apps emit + // for events without a timezone — DAVx5 sends it from Fossify + // Calendar, and rejecting it failed the whole event sync with a 400 + // (#682). Accept it and interpret the wall-clock time as UTC. + // TZID-anchored forms remain unsupported — future work when we + // tackle VTIMEZONE properly. + let has_utc_suffix = value.len() == 16 && value.ends_with('Z'); + let is_floating = value.len() == 15; + if !has_utc_suffix && !is_floating { return Err(format!( "Invalid datetime format: expected YYYYMMDDTHHMMSSZ, got {:?}", value @@ -1355,6 +1361,24 @@ SUMMARY:Weekly all-day — rescheduled\r RECURRENCE-ID;VALUE=DATE:20260112\r END:VEVENT\r END:VCALENDAR\r +"; + + /// Floating-time VEVENT — DTSTART/DTEND without the UTC 'Z' suffix + /// (RFC 5545 §3.3.5 "form #2": local time, no timezone reference). + /// This is what DAVx5 syncs from calendar apps for events created + /// without a timezone (e.g. Fossify Calendar); rejecting it failed + /// the entire event upload with a 400 (#682). + const FLOATING_TIME_EVENT: &str = "BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//OxiCloud test//EN +BEGIN:VEVENT +UID:floating-1@oxicloud.test +DTSTAMP:20260101T100000Z +DTSTART:20260831T154000 +DTEND:20260831T164000 +SUMMARY:Floating time event +END:VEVENT +END:VCALENDAR "; fn parse_ok(body: &str) -> CalendarEvent { @@ -1369,6 +1393,18 @@ END:VCALENDAR\r assert!(!ev.all_day()); } + #[test] + fn floating_time_event_parses_as_utc_wall_clock() { + // Regression (#682): '20260831T154000' used to be rejected with + // "Invalid datetime format: expected YYYYMMDDTHHMMSSZ" and the + // whole DAVx5 sync failed with HTTP 400. + let ev = parse_ok(FLOATING_TIME_EVENT); + assert_eq!(ev.summary(), "Floating time event"); + assert!(!ev.all_day()); + assert_eq!(ev.start_time().to_rfc3339(), "2026-08-31T15:40:00+00:00"); + assert_eq!(ev.end_time().to_rfc3339(), "2026-08-31T16:40:00+00:00"); + } + #[test] fn all_day_event_parses_and_flags_as_all_day() { // Regression: DTSTART;VALUE=DATE:20260201 used to fail diff --git a/src/domain/entities/session.rs b/src/domain/entities/session.rs index 7e30653a..cf2b10aa 100644 --- a/src/domain/entities/session.rs +++ b/src/domain/entities/session.rs @@ -97,6 +97,16 @@ pub struct Session { /// construction so a callsite can't forget to record it (the /// admin sessions panel filters on this). origin: SessionOrigin, + /// Wall-clock time the session was last observed serving an + /// authenticated request. Set to `created_at` at construction so + /// a freshly-minted session immediately counts as "recently + /// active" for the liveness gauges; moved forward in batches by + /// [`LastSeenTracker`](crate::infrastructure::services::last_seen_tracker) + /// via a per-30 s UNNEST-based UPDATE, so per-request writes + /// stay in-process. Distinct from `created_at` — that only moves + /// on session rotation (silent refresh), so its resolution is + /// capped at the access-token TTL. See `docs/plan/sessions.md`. + last_seen_at: DateTime, } impl Session { @@ -129,6 +139,7 @@ impl Session { oidc_sid: None, dpop_jkt: None, origin, + last_seen_at: now, } } @@ -180,6 +191,7 @@ impl Session { oidc_sid: Option, dpop_jkt: Option, origin: SessionOrigin, + last_seen_at: DateTime, ) -> Self { Self { id, @@ -195,6 +207,7 @@ impl Session { oidc_sid, dpop_jkt, origin, + last_seen_at, } } @@ -258,6 +271,10 @@ impl Session { pub fn origin(&self) -> SessionOrigin { self.origin } + + pub fn last_seen_at(&self) -> DateTime { + self.last_seen_at + } } #[cfg(test)] @@ -311,6 +328,7 @@ mod tests { None, Some("thumbprint-xyz".to_string()), SessionOrigin::Unknown, + Utc::now(), ); assert_eq!(s.dpop_jkt(), Some("thumbprint-xyz")); } diff --git a/src/domain/entities/user.rs b/src/domain/entities/user.rs index 37a0e3ed..5e2c8692 100644 --- a/src/domain/entities/user.rs +++ b/src/domain/entities/user.rs @@ -207,7 +207,7 @@ pub struct User { /// Owned decomposition of a [`User`] (mirrors `FileParts` / `FolderParts` / /// `ContactParts`). Lets a consumer MOVE the heap fields out instead of cloning /// them through the borrowing accessors — notably `image` (a data URI up to -/// 512 KiB) and `ui_preferences` (a JSON tree). See `UserDto::from` +/// 512 KiB) and `ui_preferences` (a JSON tree). See `PublicUserDto::from` /// (benches/ROUND20.md §A2). pub struct UserParts { pub id: Uuid, diff --git a/src/domain/repositories/user_repository.rs b/src/domain/repositories/user_repository.rs index 7bcb4148..5080a51b 100644 --- a/src/domain/repositories/user_repository.rs +++ b/src/domain/repositories/user_repository.rs @@ -1,6 +1,5 @@ use crate::common::errors::DomainError; use crate::domain::entities::user::{User, UserRole}; -use chrono::{DateTime, Utc}; use uuid::Uuid; #[derive(Debug, thiserror::Error)] @@ -26,49 +25,26 @@ pub enum UserRepositoryError { pub type UserRepositoryResult = Result; -/// Narrow projection for user-directory tables that do not need secrets, -/// profile pictures, or the cross-device UI-preferences document. +/// DB-computed booleans about a user that aren't fields on the +/// [`User`](crate::domain::entities::user::User) entity itself — +/// either derived from column presence (`password_hash IS NOT NULL`) +/// or from a cross-table lookup (`auth.sessions.last_seen_at` for +/// `is_online`). Companion to `User` on the list projection: the +/// repo computes both, the application layer packs them into +/// [`FullUserDto`](crate::application::dtos::user_dto::FullUserDto). /// -/// The full [`User`] row intentionally carries all of those fields for account -/// detail and the system address book. Reusing it for the paginated admin -/// table made PostgreSQL detoast and transfer an avatar of up to 512 KiB per -/// row, only for the handler to serialize it back to the browser where the -/// table never reads it. Keeping the projection explicit prevents a future -/// full-row field from silently returning to that hot path. -#[derive(Debug, Clone)] -pub struct UserListEntry { - pub id: Uuid, - pub username: Option, - pub email: String, - pub role: UserRole, - pub storage_quota_bytes: i64, - pub storage_used_bytes: i64, - pub last_login_at: Option>, - pub active: bool, - pub federation_kind: Option, - pub federation_issuer: Option, - pub is_external: bool, - /// TRUE when `auth.users.password_hash IS NOT NULL` — user has a - /// server-verifiable password on file (legacy or admin-set). - /// Distinct from `opaque_registered` (which is the zero-knowledge - /// envelope): a fully-migrated user carries BOTH — password for - /// the fallback / operator flows, envelope for the actual login. - /// A user with `has_password = false AND !opaque_registered AND - /// federation_issuer IS NULL` is passwordless — the only path in is - /// via magic-link (or, for externals, whatever grant they hold). +/// Not "admin-only" — every field ends up on `FullUserDto`, which +/// both admin AND self read. The name reflects "derived from the DB +/// row, not intrinsic to the User entity". +/// +/// See `docs/plan/userdto-refactor.md` for the design; this type +/// replaced the earlier `UserListEntry` narrow projection as of P6. +#[derive(Debug, Clone, Copy)] +pub struct UserDerivedFlags { pub has_password: bool, - /// TRUE when `auth.users.opaque_envelope IS NOT NULL` — the user - /// has completed OPAQUE registration (typically via the Phase 2 - /// silent-migration hook after a successful legacy login). Surfaced - /// on the admin user table so operators can see rollout progress - /// per-user. Admin-only exposure — see `AdminUserSummaryDto`. pub opaque_registered: bool, - /// TRUE when `auth.users.opaque_migrated_at IS NOT NULL` — the - /// user has completed at least one successful OPAQUE login. Distinct - /// from `opaque_registered` because a user can have an envelope on - /// file without having actually logged in via OPAQUE yet (e.g. - /// admin cleared the envelope, silent-migration hasn't re-run). pub opaque_migrated: bool, + pub is_online: bool, } // Conversion from UserRepositoryError to DomainError @@ -94,6 +70,20 @@ pub trait UserRepository: Send + Sync + 'static { /// Gets a user by ID async fn get_user_by_id(&self, id: Uuid) -> UserRepositoryResult; + /// Fetch the full `User` entity + the [`UserDerivedFlags`] in a + /// single query. Used by `/api/auth/me` and future admin single-user + /// views — anywhere the caller needs both the row itself AND the + /// derived booleans (`has_password`, OPAQUE flags, `is_online`) to + /// build a [`FullUserDto`](crate::application::dtos::user_dto::FullUserDto) + /// or [`SelfUserDto`](crate::application::dtos::user_dto::SelfUserDto). + /// Single query is cheaper than `get_user_by_id` + separate lookups + /// for OPAQUE state + `is_online`; the EXISTS subquery is cheap + /// thanks to the partial index `idx_sessions_last_seen_at`. + async fn get_user_with_derived_flags( + &self, + id: Uuid, + ) -> UserRepositoryResult<(User, UserDerivedFlags)>; + /// Batch-loads a set of users by id, preserving no particular order /// and silently skipping ids that don't match any row. Caller is /// responsible for de-duplicating the input vec. Returns an empty @@ -152,15 +142,22 @@ pub trait UserRepository: Send + Sync + 'static { include_external: bool, ) -> UserRepositoryResult>; - /// Lists the columns needed by compact user-management tables. Unlike - /// [`Self::list_users`], this never fetches password hashes, OIDC subjects, - /// avatars, names, locale state, or UI preferences. - async fn list_user_summaries( + /// Paginated admin user listing — full `User` entity + the derived + /// booleans (`has_password`, OPAQUE flags, `is_online`) in one wide + /// SELECT. Called by the admin service to build + /// `Vec` for `/api/admin/users` without paying two + /// round-trips per row (once for User, once for derived flags). + /// + /// Same `include_external` semantics as [`Self::list_users`]: + /// admin management UI passes `true`; every other caller passes + /// `false` so external / grant-only users stay off internal-user + /// surfaces. + async fn list_users_with_derived_flags( &self, limit: i64, offset: i64, include_external: bool, - ) -> UserRepositoryResult>; + ) -> UserRepositoryResult>; /// Searches users by username or email (SQL ILIKE) with a limit. /// See [`list_users`] for the meaning of `include_external`. diff --git a/src/infrastructure/repositories/pg/address_book_pg_repository.rs b/src/infrastructure/repositories/pg/address_book_pg_repository.rs index cd9ead3c..a061e630 100644 --- a/src/infrastructure/repositories/pg/address_book_pg_repository.rs +++ b/src/infrastructure/repositories/pg/address_book_pg_repository.rs @@ -7,6 +7,7 @@ use crate::domain::entities::contact::AddressBook; use crate::domain::repositories::address_book_repository::{ AddressBookRepository, AddressBookRepositoryResult, }; +use crate::domain::services::path_service::normalize_storage_name; pub struct AddressBookPgRepository { pool: Arc, @@ -40,6 +41,15 @@ impl AddressBookRepository for AddressBookPgRepository { &self, address_book: AddressBook, ) -> AddressBookRepositoryResult { + // NFC-normalize the caller-supplied display name at the last + // touch before bind. Same choke-point pattern as the storage.* + // repos — the entity constructor's normalization is bypassed by + // every real production path (`AddressBook::from_raw` + // reconstructs from DB bytes; `AddressBook::new` goes through + // an inbound DTO that may or may not have been touched). + // Enforcing here means every carddav write surface — DAV + // `MKCOL`, REST create — lands in NFC regardless. + let normalized_name = normalize_storage_name(address_book.name()); let row = sqlx::query( r#" INSERT INTO carddav.address_books (id, name, owner_id, description, color, is_public, created_at, updated_at) @@ -48,7 +58,7 @@ impl AddressBookRepository for AddressBookPgRepository { "# ) .bind(address_book.id()) - .bind(address_book.name()) + .bind(&normalized_name) .bind(address_book.owner_id()) .bind(address_book.description()) .bind(address_book.color()) @@ -76,6 +86,8 @@ impl AddressBookRepository for AddressBookPgRepository { &self, address_book: AddressBook, ) -> AddressBookRepositoryResult { + // NFC-normalize on rename — see `create_address_book` for the why. + let normalized_name = normalize_storage_name(address_book.name()); let now = Utc::now(); let row = sqlx::query( r#" @@ -85,7 +97,7 @@ impl AddressBookRepository for AddressBookPgRepository { RETURNING id, name, owner_id, description, color, is_public, created_at, updated_at "#, ) - .bind(address_book.name()) + .bind(&normalized_name) .bind(address_book.description()) .bind(address_book.color()) .bind(address_book.is_public()) diff --git a/src/infrastructure/repositories/pg/blob_reference_sources.rs b/src/infrastructure/repositories/pg/blob_reference_sources.rs new file mode 100644 index 00000000..c2b2fa39 --- /dev/null +++ b/src/infrastructure/repositories/pg/blob_reference_sources.rs @@ -0,0 +1,632 @@ +//! The two implicit blob-reference sources, made explicit. +//! +//! Before this module, "who references this hash" lived as hardcoded SQL +//! inside `blobs_consistency`'s refcount recompute and `dedup_gc`'s reap +//! predicate. These two implementations reproduce that SQL **exactly** — +//! the fragments below sum to today's `actual_ref_count` expression — so +//! the registry can be wired in without changing any observed count. +//! +//! See `docs/plan/derived-blobs.md` and +//! [`crate::application::ports::blob_reference_ports`]. + +use std::sync::Arc; + +use async_trait::async_trait; +use sqlx::{PgPool, Row}; +use uuid::Uuid; + +use crate::application::ports::blob_reference_ports::{ + BlobReferenceRegistry, BlobReferenceSource, RefLevel, +}; +use crate::domain::errors::DomainError; + +/// Aliases used inside the emitted fragments. +/// +/// Deliberately distinct from the aliases the sweeps use for their outer +/// row (`b` for `storage.blobs`, `m` for `storage.chunk_manifests`): a +/// fragment reusing `m` would shadow the outer alias in the manifest-level +/// sweep and silently correlate against itself. +const FILES_ALIAS: &str = "cnt_f"; +const MANIFEST_ALIAS: &str = "cnt_m"; +const DERIVED_ALIAS: &str = "cnt_d"; +const ATTACHED_ALIAS: &str = "cnt_a"; + +/// Fragment for [`FilesReferenceSource`], as a free function so the SQL +/// shape can be tested without constructing a pool — it is a property of +/// the module, not of an instance. +fn files_ref_sql(level: RefLevel, outer_hash_expr: &str) -> Option { + let f = FILES_ALIAS; + match level { + // Legacy whole-file blobs only — CDC files are counted at the + // manifest level, and counting them here too would double up on the + // single-chunk hash alias. + RefLevel::Chunk => Some(format!( + "(SELECT COUNT(*) FROM storage.files {f} + WHERE {f}.blob_hash = {outer_hash_expr} + AND NOT EXISTS ( + SELECT 1 FROM storage.chunk_manifests {MANIFEST_ALIAS} + WHERE {MANIFEST_ALIAS}.file_hash = {f}.blob_hash + ))" + )), + RefLevel::Manifest => Some(format!( + "(SELECT COUNT(*) FROM storage.files {f} + WHERE {f}.blob_hash = {outer_hash_expr})" + )), + } +} + +/// Short-circuiting existence form of [`files_ref_sql`]. +/// +/// `dedup_gc` evaluates this per candidate manifest, so counting every +/// referrer where existence would do is a real cost on a heavily-deduplicated +/// blob. This is also the exact shape the reap predicate used before the +/// registry existed, so wiring it in changes no plan. +fn files_exists_sql(level: RefLevel, outer_hash_expr: &str) -> Option { + let f = FILES_ALIAS; + match level { + RefLevel::Chunk => Some(format!( + "EXISTS (SELECT 1 FROM storage.files {f} \ + WHERE {f}.blob_hash = {outer_hash_expr} \ + AND NOT EXISTS (SELECT 1 FROM storage.chunk_manifests {MANIFEST_ALIAS} \ + WHERE {MANIFEST_ALIAS}.file_hash = {f}.blob_hash))" + )), + RefLevel::Manifest => Some(format!( + "EXISTS (SELECT 1 FROM storage.files {f} WHERE {f}.blob_hash = {outer_hash_expr})" + )), + } +} + +/// Short-circuiting existence form of [`chunks_ref_sql`]. +/// +/// Same motivation as [`files_exists_sql`], and it now matters more: this +/// fragment sits in `dedup_gc`'s **phase-2 reap guard**, evaluated per +/// candidate blob row. Without the override the trait default wraps the +/// counting form as `(SELECT COUNT(*) …) > 0`, which scans every manifest +/// listing the chunk before comparing — a heavily-deduplicated chunk is +/// exactly the case where that is most expensive and least necessary. +fn chunks_exists_sql(level: RefLevel, outer_hash_expr: &str) -> Option { + match level { + RefLevel::Chunk => { + let m = MANIFEST_ALIAS; + Some(format!( + "EXISTS (SELECT 1 FROM storage.chunk_manifests {m} \ + WHERE {outer_hash_expr} = ANY({m}.chunk_hashes))" + )) + } + RefLevel::Manifest => None, + } +} + +/// Fragment for [`ChunksReferenceSource`]. See [`files_ref_sql`]. +fn chunks_ref_sql(level: RefLevel, outer_hash_expr: &str) -> Option { + match level { + RefLevel::Chunk => { + let m = MANIFEST_ALIAS; + Some(format!( + "(SELECT COUNT(*) FROM storage.chunk_manifests {m} + WHERE {outer_hash_expr} = ANY({m}.chunk_hashes))" + )) + } + // A manifest is never referenced by another manifest. + RefLevel::Manifest => None, + } +} + +/// Fragment for [`ContentDerivedReferenceSource`]. +/// +/// **Manifest level only.** A derived artifact's `blob_hash` names a Blob +/// (its own manifest), never a chunk. Contributing at the chunk level would +/// double-count, because a thumbnail is almost always single-chunk and its +/// manifest hash therefore equals its lone chunk's hash. +fn content_derived_ref_sql(level: RefLevel, outer_hash_expr: &str) -> Option { + match level { + RefLevel::Chunk => None, + RefLevel::Manifest => Some(format!( + "(SELECT COUNT(*) FROM storage.content_derived_blobs {DERIVED_ALIAS} \ + WHERE {DERIVED_ALIAS}.blob_hash = {outer_hash_expr})" + )), + } +} + +/// Short-circuiting existence form, used by `dedup_gc`'s reap predicate. +fn content_derived_exists_sql(level: RefLevel, outer_hash_expr: &str) -> Option { + match level { + RefLevel::Chunk => None, + RefLevel::Manifest => Some(format!( + "EXISTS (SELECT 1 FROM storage.content_derived_blobs {DERIVED_ALIAS} \ + WHERE {DERIVED_ALIAS}.blob_hash = {outer_hash_expr})" + )), + } +} + +/// Fragment for [`FileAttachedReferenceSource`]. +/// +/// **Manifest level only**, for the same reason as the derived source: an +/// attached artifact's `blob_hash` names a Blob, never a chunk, and these are +/// almost always single-chunk — so contributing at the chunk level would +/// double-count against the aliased hash. +fn file_attached_ref_sql(level: RefLevel, outer_hash_expr: &str) -> Option { + match level { + RefLevel::Chunk => None, + RefLevel::Manifest => Some(format!( + "(SELECT COUNT(*) FROM storage.file_attached_blobs {ATTACHED_ALIAS} \ + WHERE {ATTACHED_ALIAS}.blob_hash = {outer_hash_expr})" + )), + } +} + +/// Short-circuiting existence form, used by `dedup_gc`'s reap predicate. +fn file_attached_exists_sql(level: RefLevel, outer_hash_expr: &str) -> Option { + match level { + RefLevel::Chunk => None, + RefLevel::Manifest => Some(format!( + "EXISTS (SELECT 1 FROM storage.file_attached_blobs {ATTACHED_ALIAS} \ + WHERE {ATTACHED_ALIAS}.blob_hash = {outer_hash_expr})" + )), + } +} + +/// Every built-in blob-reference source, in one place. +/// +/// THE definition of "what references a blob". `DedupService::new` uses it +/// as its construction default and hands it to the consistency jobs via +/// `reference_registry()`, so GC and the sweeps cannot disagree — and the +/// golden tests that pin the generated SQL exercise the same set production +/// runs, rather than a test-local approximation of it. +pub fn built_in_registry(pool: Arc) -> BlobReferenceRegistry { + let mut registry = BlobReferenceRegistry::new(); + registry.register(Arc::new(FilesReferenceSource::new(pool.clone()))); + registry.register(Arc::new(ChunksReferenceSource::new(pool.clone()))); + // Registered before anything writes a derived blob: dedup_gc's reap + // predicate must already know this table exists, or the first sweep + // after the first thumbnail deletes it. + registry.register(Arc::new(ContentDerivedReferenceSource::new(pool.clone()))); + // Same rule as above: registered before the first attachment is written, + // so dedup_gc's reap predicate already knows the table exists. + registry.register(Arc::new(FileAttachedReferenceSource::new(pool))); + registry +} + +// ─── storage.files ─────────────────────────────────────────────────────── + +/// References held by `storage.files.blob_hash`. +/// +/// Contributes at **both** levels, which is why `RefLevel` is a parameter +/// rather than a property of the source: +/// +/// * [`RefLevel::Manifest`] — a CDC file's `blob_hash` names a manifest. +/// * [`RefLevel::Chunk`] — a pre-CDC legacy file, whose `blob_hash` names a +/// whole-file blob with no manifest behind it. The `NOT EXISTS` guard is +/// load-bearing: for a single-chunk file the whole-file hash *equals* its +/// lone chunk's hash, so without it the row would be counted at both +/// levels. +pub struct FilesReferenceSource { + pool: Arc, +} + +impl FilesReferenceSource { + pub fn new(pool: Arc) -> Self { + Self { pool } + } +} + +#[async_trait] +impl BlobReferenceSource for FilesReferenceSource { + fn source_name(&self) -> &'static str { + "files" + } + + fn ref_count_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + files_ref_sql(level, outer_hash_expr) + } + + fn ref_exists_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + files_exists_sql(level, outer_hash_expr) + } + + async fn count_references(&self, blob_hash: &str) -> Result { + // No level split here: the question is "how many file rows name this + // exact hash", and a hash names either a manifest or a legacy blob, + // never both at once from the caller's point of view. + let n: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM storage.files WHERE blob_hash = $1") + .bind(blob_hash) + .fetch_one(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("BlobRefSource", format!("files count: {e}")) + })?; + Ok(n.max(0) as u64) + } + + async fn list_referenced_blobs( + &self, + cursor: Option>, + limit: usize, + ) -> Result<(Vec, Option>), DomainError> { + // Paged by the file's own PK so the cursor is stable under concurrent + // inserts; `blob_hash` is not unique and would skip or repeat rows. + let after: Option = match cursor { + Some(bytes) => Some(decode_uuid_cursor(&bytes)?), + None => None, + }; + + let rows = sqlx::query( + "SELECT id, blob_hash FROM storage.files + WHERE ($1::uuid IS NULL OR id > $1) + ORDER BY id + LIMIT $2", + ) + .bind(after) + .bind(limit as i64) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("BlobRefSource", format!("files page: {e}")))?; + + let next = rows + .last() + .map(|r| r.get::("id").as_bytes().to_vec()) + .filter(|_| rows.len() == limit); + let hashes = rows + .iter() + .map(|r| r.get::("blob_hash")) + .collect(); + Ok((hashes, next)) + } +} + +// ─── storage.chunk_manifests ───────────────────────────────────────────── + +/// References held by `storage.chunk_manifests.chunk_hashes[]`. +/// +/// Chunk level only — a manifest never references another manifest, so +/// [`RefLevel::Manifest`] yields `None` and this source contributes nothing +/// to the manifest recompute. +pub struct ChunksReferenceSource { + pool: Arc, +} + +impl ChunksReferenceSource { + pub fn new(pool: Arc) -> Self { + Self { pool } + } +} + +#[async_trait] +impl BlobReferenceSource for ChunksReferenceSource { + fn source_name(&self) -> &'static str { + "chunks" + } + + fn ref_count_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + chunks_ref_sql(level, outer_hash_expr) + } + + fn ref_exists_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + chunks_exists_sql(level, outer_hash_expr) + } + + async fn count_references(&self, blob_hash: &str) -> Result { + let n: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM storage.chunk_manifests WHERE $1 = ANY(chunk_hashes)", + ) + .bind(blob_hash) + .fetch_one(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("BlobRefSource", format!("chunks count: {e}")))?; + Ok(n.max(0) as u64) + } + + async fn list_referenced_blobs( + &self, + cursor: Option>, + limit: usize, + ) -> Result<(Vec, Option>), DomainError> { + // Paged by the manifest PK, not by the unnested chunk hash: a single + // manifest expands to many hashes, so the page boundary has to fall + // between manifests or the cursor cannot be resumed unambiguously. + let after: Option = match cursor { + Some(bytes) => Some(String::from_utf8(bytes).map_err(|e| { + DomainError::internal_error("BlobRefSource", format!("bad chunk cursor: {e}")) + })?), + None => None, + }; + + let rows = sqlx::query( + "SELECT file_hash, chunk_hashes FROM storage.chunk_manifests + WHERE ($1::text IS NULL OR file_hash > $1) + ORDER BY file_hash + LIMIT $2", + ) + .bind(after) + .bind(limit as i64) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("BlobRefSource", format!("chunks page: {e}")))?; + + let next = rows + .last() + .map(|r| r.get::("file_hash").into_bytes()) + .filter(|_| rows.len() == limit); + let hashes = rows + .iter() + .flat_map(|r| r.get::, _>("chunk_hashes")) + .collect(); + Ok((hashes, next)) + } +} + +fn decode_uuid_cursor(bytes: &[u8]) -> Result { + let raw: [u8; 16] = bytes.try_into().map_err(|_| { + DomainError::internal_error( + "BlobRefSource", + format!("bad uuid cursor: expected 16 bytes, got {}", bytes.len()), + ) + })?; + Ok(Uuid::from_bytes(raw)) +} + +// ─── storage.content_derived_blobs ─────────────────────────────────────── + +/// References held by `storage.content_derived_blobs.blob_hash` — the +/// DERIVED artifact, not the source it came from. +/// +/// **`source_hash` is deliberately not a reference.** It is a dependent +/// pointer: the source Blob is kept alive by the file that owns it, and when +/// that Blob is reaped these rows go with it. Counting `source_hash` here +/// would pin every source Blob for as long as a thumbnail existed. +pub struct ContentDerivedReferenceSource { + pool: Arc, +} + +impl ContentDerivedReferenceSource { + pub fn new(pool: Arc) -> Self { + Self { pool } + } +} + +#[async_trait] +impl BlobReferenceSource for ContentDerivedReferenceSource { + fn source_name(&self) -> &'static str { + "content_derived" + } + + fn ref_count_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + content_derived_ref_sql(level, outer_hash_expr) + } + + fn ref_exists_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + content_derived_exists_sql(level, outer_hash_expr) + } + + async fn count_references(&self, blob_hash: &str) -> Result { + let n: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM storage.content_derived_blobs WHERE blob_hash = $1", + ) + .bind(blob_hash) + .fetch_one(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("BlobRefSource", format!("derived count: {e}")))?; + Ok(n.max(0) as u64) + } + + async fn list_referenced_blobs( + &self, + cursor: Option>, + limit: usize, + ) -> Result<(Vec, Option>), DomainError> { + // Paged by `blob_hash` itself — unlike files it IS the value we + // return, and DISTINCT keeps a Blob shared by several variants from + // appearing more than once per page. + // + // `IS NOT NULL` is load-bearing, not defensive. A NEGATIVE row — + // "this content is not worth transcoding" — carries a NULL + // blob_hash, and this query decodes into `String`, so the first one + // ever written would fail the decode and take the whole enumeration + // down. It would also be wrong if it decoded: a negative row holds + // no reference on any Blob, which is exactly why the counting forms + // above (`WHERE blob_hash = `) already exclude it for free — + // NULL equals nothing. + let after: Option = match cursor { + Some(bytes) => Some(String::from_utf8(bytes).map_err(|e| { + DomainError::internal_error("BlobRefSource", format!("bad derived cursor: {e}")) + })?), + None => None, + }; + + let rows: Vec<(String,)> = sqlx::query_as( + "SELECT DISTINCT blob_hash FROM storage.content_derived_blobs + WHERE blob_hash IS NOT NULL + AND ($1::text IS NULL OR blob_hash > $1) + ORDER BY blob_hash + LIMIT $2", + ) + .bind(after) + .bind(limit as i64) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("BlobRefSource", format!("derived page: {e}")))?; + + let next = rows + .last() + .map(|(h,)| h.clone().into_bytes()) + .filter(|_| rows.len() == limit); + Ok((rows.into_iter().map(|(h,)| h).collect(), next)) + } +} + +// ─── storage.file_attached_blobs ───────────────────────────────────────── + +/// References held by `storage.file_attached_blobs.blob_hash` — bytes a user +/// supplied for one specific file. +/// +/// Structurally the twin of [`ContentDerivedReferenceSource`]: same level, +/// same shape, different table. The difference that matters is upstream — the +/// row is keyed by `file_id` rather than by content, so the same bytes +/// attached to two files are two rows and therefore two references. Dedup +/// still applies to the bytes; what must not be shared is the mapping. +/// +/// `file_id` is deliberately not a reference at this layer: it is an +/// `ON DELETE CASCADE` foreign key, so the row disappears with the file, and +/// the blob reference it held is released by the owning service's +/// `on_file_deleted` hook. +pub struct FileAttachedReferenceSource { + pool: Arc, +} + +impl FileAttachedReferenceSource { + pub fn new(pool: Arc) -> Self { + Self { pool } + } +} + +#[async_trait] +impl BlobReferenceSource for FileAttachedReferenceSource { + fn source_name(&self) -> &'static str { + "file_attached" + } + + fn ref_count_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + file_attached_ref_sql(level, outer_hash_expr) + } + + fn ref_exists_sql(&self, level: RefLevel, outer_hash_expr: &str) -> Option { + file_attached_exists_sql(level, outer_hash_expr) + } + + async fn count_references(&self, blob_hash: &str) -> Result { + let n: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM storage.file_attached_blobs WHERE blob_hash = $1", + ) + .bind(blob_hash) + .fetch_one(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("BlobRefSource", format!("attached count: {e}")) + })?; + Ok(n.max(0) as u64) + } + + async fn list_referenced_blobs( + &self, + cursor: Option>, + limit: usize, + ) -> Result<(Vec, Option>), DomainError> { + // Paged by `blob_hash`, same as the derived source: it IS the value + // returned, and DISTINCT collapses one Blob attached to several files. + let after: Option = match cursor { + Some(bytes) => Some(String::from_utf8(bytes).map_err(|e| { + DomainError::internal_error("BlobRefSource", format!("bad attached cursor: {e}")) + })?), + None => None, + }; + + let rows: Vec<(String,)> = sqlx::query_as( + "SELECT DISTINCT blob_hash FROM storage.file_attached_blobs + WHERE ($1::text IS NULL OR blob_hash > $1) + ORDER BY blob_hash + LIMIT $2", + ) + .bind(after) + .bind(limit as i64) + .fetch_all(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("BlobRefSource", format!("attached page: {e}")))?; + + let next = rows + .last() + .map(|(h,)| h.clone().into_bytes()) + .filter(|_| rows.len() == limit); + Ok((rows.into_iter().map(|(h,)| h).collect(), next)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::application::ports::blob_reference_ports::BlobReferenceRegistry; + + /// The registry sums whatever the sources emit; these helpers exercise the + /// same code path without needing a pool, since `ref_count_sql` is pure. + fn summed(level: RefLevel, outer: &str) -> String { + let frags: Vec = [files_ref_sql(level, outer), chunks_ref_sql(level, outer)] + .into_iter() + .flatten() + .collect(); + if frags.is_empty() { + "0".to_string() + } else { + frags.join("\n + ") + } + } + + /// The chunk-level expression must reproduce the two terms + /// `blobs_consistency` inlines today: legacy-only files (guarded by + /// NOT EXISTS) plus manifests citing the chunk. + #[test] + fn chunk_level_reproduces_todays_two_terms() { + let expr = summed(RefLevel::Chunk, "b.hash"); + assert!(expr.contains("storage.files"), "{expr}"); + assert!( + expr.contains("NOT EXISTS"), + "legacy term must keep the CDC guard: {expr}" + ); + assert!( + expr.contains("= ANY(cnt_m.chunk_hashes)"), + "chunk term missing: {expr}" + ); + assert!(expr.contains("b.hash"), "must correlate on the outer row"); + assert!(expr.contains('+'), "both terms must be summed: {expr}"); + } + + /// Only `storage.files` references a manifest, so the manifest-level + /// expression is the single files term with no `NOT EXISTS` guard — the + /// guard exists to keep CDC rows *out* of the chunk level, and applying + /// it here would count nothing at all. + #[test] + fn manifest_level_is_files_only_and_unguarded() { + let expr = summed(RefLevel::Manifest, "m.file_hash"); + assert!(expr.contains("storage.files"), "{expr}"); + assert!(!expr.contains("NOT EXISTS"), "{expr}"); + assert!( + !expr.contains("chunk_hashes"), + "chunks must not contribute at manifest level: {expr}" + ); + assert!(!expr.contains('+'), "only one source contributes: {expr}"); + assert!(expr.contains("m.file_hash")); + } + + /// Fragments must not use the aliases the sweeps use for their outer row + /// (`b` for storage.blobs, `m` for chunk_manifests), or the manifest sweep + /// would shadow its own alias and silently correlate against itself. + #[test] + fn fragments_avoid_outer_row_aliases() { + for level in RefLevel::ALL { + let expr = summed(level, "m.file_hash"); + for bad in [ + "storage.files f", + "storage.files b", + "chunk_manifests m ", + "chunk_manifests b", + ] { + assert!(!expr.contains(bad), "alias collision at {level:?}: {expr}"); + } + } + } + + /// A source declining a level must drop out of the sum entirely, which is + /// what keeps manifest-only tables out of the chunk recompute where the + /// single-chunk hash alias would double-count them. + #[test] + fn chunks_source_declines_manifest_level() { + assert!(chunks_ref_sql(RefLevel::Manifest, "m.file_hash").is_none()); + assert!(chunks_ref_sql(RefLevel::Chunk, "b.hash").is_some()); + } + + /// Guards the registry contract the sweeps rely on: an empty level still + /// yields a valid scalar expression. + #[test] + fn empty_registry_yields_zero_literal() { + let r = BlobReferenceRegistry::new(); + assert_eq!(r.ref_count_expr(RefLevel::Manifest, "m.file_hash"), "0"); + } +} diff --git a/src/infrastructure/repositories/pg/calendar_pg_repository.rs b/src/infrastructure/repositories/pg/calendar_pg_repository.rs index fe0c213f..423c3ee1 100644 --- a/src/infrastructure/repositories/pg/calendar_pg_repository.rs +++ b/src/infrastructure/repositories/pg/calendar_pg_repository.rs @@ -7,6 +7,7 @@ use crate::domain::entities::calendar::Calendar; use crate::domain::repositories::calendar_repository::{ CalendarRepository, CalendarRepositoryResult, }; +use crate::domain::services::path_service::normalize_storage_name; pub struct CalendarPgRepository { pool: Arc, @@ -38,6 +39,15 @@ impl CalendarPgRepository { impl CalendarRepository for CalendarPgRepository { async fn create_calendar(&self, calendar: Calendar) -> CalendarRepositoryResult { + // NFC-normalize the caller-supplied display name at the last + // touch before bind — same choke-point pattern the storage.files + // / storage.folders repos use (see docs/plan/nfc-normalization.md + // / migrate.rs module doc). macOS CalDAV clients emit NFD in the + // display-name field just as Finder does in the filename field; + // NC-desktop / Thunderbird would then miss the calendar on their + // NFC-normalized lookup path. Same class of bug as + // AtalayaLabs/OxiCloud#706, different table. + let normalized_name = normalize_storage_name(calendar.name()); let row = sqlx::query( r#" INSERT INTO caldav.calendars (id, name, owner_id, description, color, is_public, created_at, updated_at) @@ -46,7 +56,7 @@ impl CalendarRepository for CalendarPgRepository { "# ) .bind(calendar.id()) - .bind(calendar.name()) + .bind(&normalized_name) .bind(calendar.owner_id()) .bind(calendar.description()) .bind(calendar.color()) @@ -75,6 +85,8 @@ impl CalendarRepository for CalendarPgRepository { } async fn update_calendar(&self, calendar: Calendar) -> CalendarRepositoryResult { + // NFC-normalize on rename — see `create_calendar` for the why. + let normalized_name = normalize_storage_name(calendar.name()); let now = Utc::now(); let row = sqlx::query( r#" @@ -84,7 +96,7 @@ impl CalendarRepository for CalendarPgRepository { RETURNING id, name, owner_id, description, color, is_public, created_at, updated_at "#, ) - .bind(calendar.name()) + .bind(&normalized_name) .bind(calendar.description()) .bind(calendar.color()) .bind(false) // is_public doesn't exist as a field diff --git a/src/infrastructure/repositories/pg/contact_group_pg_repository.rs b/src/infrastructure/repositories/pg/contact_group_pg_repository.rs index 02383cc0..9e0dd3ee 100644 --- a/src/infrastructure/repositories/pg/contact_group_pg_repository.rs +++ b/src/infrastructure/repositories/pg/contact_group_pg_repository.rs @@ -11,6 +11,7 @@ use crate::domain::entities::contact::{Contact, ContactGroup}; use crate::domain::repositories::contact_repository::{ ContactGroupRepository, ContactRepositoryResult, }; +use crate::domain::services::path_service::normalize_storage_name; pub struct ContactGroupPgRepository { pool: Arc, @@ -24,12 +25,19 @@ impl ContactGroupPgRepository { impl ContactGroupRepository for ContactGroupPgRepository { async fn create_group(&self, group: ContactGroup) -> ContactRepositoryResult { + // NFC-normalize at the last touch before bind — same choke-point + // pattern as the storage.* / caldav.* / carddav.address_books + // repos. Group display names on macOS Contacts sync as NFD + // (Address Book pushes decomposed forms in vCard KIND=group); + // NC-desktop / Thunderbird would then miss the group on their + // NFC-normalized lookup. + let normalized_name = normalize_storage_name(group.name()); sqlx::query( "INSERT INTO carddav.contact_groups (id, address_book_id, name, created_at, updated_at) VALUES ($1, $2, $3, $4, $5)" ) .bind(group.id()) .bind(group.address_book_id()) - .bind(group.name()) + .bind(&normalized_name) .bind(group.created_at()) .bind(group.updated_at()) .execute(self.pool.as_ref()) @@ -40,8 +48,10 @@ impl ContactGroupRepository for ContactGroupPgRepository { } async fn update_group(&self, group: ContactGroup) -> ContactRepositoryResult { + // NFC-normalize on rename — see `create_group` for the why. + let normalized_name = normalize_storage_name(group.name()); sqlx::query("UPDATE carddav.contact_groups SET name = $1, updated_at = $2 WHERE id = $3") - .bind(group.name()) + .bind(&normalized_name) .bind(Utc::now()) .bind(group.id()) .execute(self.pool.as_ref()) diff --git a/src/infrastructure/repositories/pg/drive_pg_repository.rs b/src/infrastructure/repositories/pg/drive_pg_repository.rs index e7f51ede..e207bc00 100644 --- a/src/infrastructure/repositories/pg/drive_pg_repository.rs +++ b/src/infrastructure/repositories/pg/drive_pg_repository.rs @@ -19,6 +19,7 @@ use crate::domain::entities::drive::{Drive, DriveKind}; use crate::domain::repositories::drive_repository::{ DriveRepository, DriveRepositoryError, DriveWithRootName, }; +use crate::domain::services::path_service::normalize_storage_name; /// Decode a `d.policies` JSONB column straight into `DrivePolicies` via /// `sqlx::types::Json` — a single `serde_json::from_slice` over the raw JSONB @@ -395,6 +396,13 @@ impl DriveRepository for DrivePgRepository { quota_bytes: Option, granted_by: Uuid, ) -> Result { + // NFC-normalize the admin-supplied shared-drive root name — same + // reasoning as `folder_db_repository::create_folder`. Even though + // this write is admin-only (not end-user-driven), the field feeds + // straight into `storage.folders.name` and WebDAV path lookups + // against it must match what NFC-normalizing clients send. + let name = normalize_storage_name(name); + // Same four-write transaction shape as `create_personal_drive_atomic` // (see that method for the why-not-CTE explanation). Differences: // - `kind='shared'`, `default_for_user=NULL`. diff --git a/src/infrastructure/repositories/pg/external_mount_repository.rs b/src/infrastructure/repositories/pg/external_mount_repository.rs index 366f1294..90dc683c 100644 --- a/src/infrastructure/repositories/pg/external_mount_repository.rs +++ b/src/infrastructure/repositories/pg/external_mount_repository.rs @@ -13,6 +13,7 @@ use crate::application::ports::external_mount_ports::{ ExternalMountRecord, ExternalMountRepositoryPort, NewExternalMount, }; use crate::domain::errors::DomainError; +use crate::domain::services::path_service::normalize_storage_name; /// PostgreSQL implementation of [`ExternalMountRepositoryPort`]. pub struct ExternalMountPgRepository { @@ -93,6 +94,14 @@ impl ExternalMountRepositoryPort for ExternalMountPgRepository { } async fn create(&self, mount: &NewExternalMount) -> Result<(), DomainError> { + // NFC-normalize the admin-supplied display label at the last + // touch before bind. Admin-facing (not end-user drag-drop) so + // NFD is unlikely, but the invariant matches the storage.* + // pattern — the sibling folder row (created via + // `folder_db_repository::create_folder`, which already + // normalizes) and this admin label should stay byte-consistent + // on any table. + let normalized_name = normalize_storage_name(&mount.name); sqlx::query( "INSERT INTO storage.external_mounts (mount_folder_id, kind, config, name, owner_id, read_only) @@ -101,7 +110,7 @@ impl ExternalMountRepositoryPort for ExternalMountPgRepository { .bind(mount.mount_folder_id) .bind(&mount.kind) .bind(&mount.config) - .bind(&mount.name) + .bind(&normalized_name) .bind(mount.owner_id) .bind(mount.read_only) .execute(self.pool.as_ref()) diff --git a/src/infrastructure/repositories/pg/face_pg_repository.rs b/src/infrastructure/repositories/pg/face_pg_repository.rs index a9a4e12b..260c82df 100644 --- a/src/infrastructure/repositories/pg/face_pg_repository.rs +++ b/src/infrastructure/repositories/pg/face_pg_repository.rs @@ -47,8 +47,17 @@ fn embedding_to_bytes(e: &[f32]) -> Vec { } fn bytes_to_embedding(b: &[u8]) -> Vec { - b.chunks_exact(4) - .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + // `as_chunks::<4>` (stable since Rust 1.88) hands back `&[[u8; 4]]` + // typed at the array level, so the closure gets a `&[u8; 4]` and + // the `[c[0], c[1], c[2], c[3]]` array-copy dance from the old + // `chunks_exact(4)` shape collapses to a plain deref. Any trailing + // bytes that aren't a multiple of 4 land in `.1` and are dropped + // — same semantics as `chunks_exact` which iterated only the + // aligned prefix. + b.as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) .collect() } diff --git a/src/infrastructure/repositories/pg/file_blob_write_repository.rs b/src/infrastructure/repositories/pg/file_blob_write_repository.rs index b84a5378..591b224d 100644 --- a/src/infrastructure/repositories/pg/file_blob_write_repository.rs +++ b/src/infrastructure/repositories/pg/file_blob_write_repository.rs @@ -17,6 +17,7 @@ use crate::application::dtos::display_helpers::category_order_for; use crate::application::ports::storage_ports::{CopyFolderTreeResult, FileWritePort}; use crate::common::errors::DomainError; use crate::domain::entities::file::File; +use crate::domain::services::path_service::{normalize_storage_name, normalize_storage_name_owned}; use super::transaction_utils::retry_on_deadlock; use crate::infrastructure::services::dedup_service::DedupService; @@ -130,9 +131,11 @@ impl FileBlobWriteRepository { DomainError::internal_error("FileBlobWrite", format!("parent lookup: {e}")) })? .ok_or_else(|| DomainError::not_found("Folder", fid)), - None => Err(DomainError::internal_error( - "FileBlobWrite", - "folder_id is required to determine the target drive", + // Same reasoning as the owner lookup below: caller error, not + // server error. + None => Err(DomainError::validation_error( + "folder_id is required: the destination folder determines the \ + target drive", )), } } @@ -279,6 +282,16 @@ impl FileBlobWriteRepository { size: u64, caller_id: Uuid, ) -> Result { + // NFC-normalize at the last touch before the DB bind. Same + // reasoning as `folder_db_repository::create_folder`: every write + // surface that lands here — REST multipart upload, by-hash instant + // upload, chunked-upload complete, WOPI create-fallback, WebDAV + // PUT, NC PUT, NC chunked-upload assemble — passes raw client + // bytes. macOS Finder emits NFD; canonicalising once here closes + // every audited entry-point at one choke-point. `is_nfc_quick` + // fast path is one table-lookup for the ~99% of names already NFC. + let name = normalize_storage_name_owned(name); + // Root files have no parent folder to derive an owner from — keep the // previous resolve_user_id(None) contract (release the ref, error out). let Some(fid) = folder_id.as_deref() else { @@ -289,9 +302,15 @@ impl FileBlobWriteRepository { rollback_err ); } - return Err(DomainError::internal_error( - "FileBlobWrite", - "folder_id is required to determine file owner", + // A missing required field is the caller's error, not the + // server's. As `internal_error` this surfaced as 500 / + // `error_type: Internal Error`, which the SPA cannot tell apart + // from the server breaking — so a malformed upload looked like an + // outage. The OpenAPI body description called the field optional, + // which is how it came to be omitted in the first place. + return Err(DomainError::validation_error( + "folder_id is required: the destination folder determines the \ + file's owner and drive", )); }; @@ -596,8 +615,25 @@ impl FileWritePort for FileBlobWriteRepository { new_name: Option<&str>, caller_id: Uuid, ) -> Result { - // Atomic CTE: read source file → insert new row with same blob_hash → increment ref_count. - // Single round-trip; blob content is NOT copied (dedup makes this zero-copy). + // Two statements in one transaction: insert the new row (same + // blob_hash — blob content is never copied, dedup makes this + // zero-copy), then run the shared satellite fan-out. + // + // `storage.copy_file_satellites` is the single home for everything + // that follows a file on copy — dead properties and the + // manifest-aware blob reference — shared with + // `storage.copy_folder_tree`. Two sites implementing that + // separately is what let the tree path ship a version that missed + // manifests entirely (migration `20261019000000`). + // + // It cannot be a CTE arm: data-modifying CTEs all observe the same + // snapshot, so a function called alongside the INSERT would not see + // the new `storage.files` row it needs to read `blob_hash` from, + // and the dead-property INSERT would fail its foreign key. Hence a + // real transaction — which also fixes the reference being + // best-effort before: a failed `add_reference` used to log a + // warning and leave a copy holding no reference at all, the exact + // shape that gets its content reaped. // // §14: `created_by = $4 = updated_by = caller_id` — the caller // authored this copy. The previous binding used @@ -605,10 +641,19 @@ impl FileWritePort for FileBlobWriteRepository { // folder's owner as the author when Adam copied a file into // Alice's folder. let target_fid = target_folder_id.clone(); - let rename_to = new_name.map(|s| s.to_string()); + // NFC-normalize the destination name at the last touch before the + // bind. `new_name = None` means "keep the source's stored name" — + // that path is already normalized (either by an earlier write here + // or, for pre-fix rows, deliberately left as-is per operator + // decision to not touch historical NFD content). Only fresh + // client-supplied `new_name` needs the pass; WebDAV `COPY` with a + // Destination header renaming a file is the canonical caller. + let rename_to = new_name.map(normalize_storage_name); - let row = retry_on_deadlock("files.copy", || { - sqlx::query_as::< + let row = retry_on_deadlock("files.copy", || async { + let mut tx = self.pool.begin().await?; + + let row = sqlx::query_as::< _, ( String, @@ -662,20 +707,6 @@ impl FileWritePort for FileBlobWriteRepository { blob_hash, created_by, updated_by - ), - -- RFC 4918 §8.8 — dead properties MUST be duplicated on - -- COPY. With the id-keyed store (migration - -- 20260830000001) this is a single batch INSERT keyed on - -- the new file's id. Runs in the same query as the file - -- INSERT so either both land or neither does — atomic - -- by virtue of being one statement. - dead_prop_copy AS ( - INSERT INTO storage.webdav_dead_properties - (file_id, namespace, local_name, value) - SELECT (SELECT id FROM new_file), - dp.namespace, dp.local_name, dp.value - FROM storage.webdav_dead_properties dp - WHERE dp.file_id = $1::uuid ) SELECT id_text, name, folder_id, size, mime_type, created_at, updated_at, @@ -687,7 +718,22 @@ impl FileWritePort for FileBlobWriteRepository { .bind(&target_fid) .bind(&rename_to) .bind(caller_id) - .fetch_optional(self.pool.as_ref()) + .fetch_optional(&mut *tx) + .await?; + + if let Some(ref new_row) = row { + // `new_row.0` is the new file's id as text; PG casts it. + sqlx::query( + "SELECT storage.copy_file_satellites(ARRAY[$1::uuid], ARRAY[$2::uuid])", + ) + .bind(file_id) + .bind(&new_row.0) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + Ok(row) }) .await .map_err(|e| { @@ -705,14 +751,8 @@ impl FileWritePort for FileBlobWriteRepository { let blob_hash = &row.7; - // Increment blob reference count (best-effort; INSERT already succeeded) - if let Err(e) = self.dedup.add_reference(blob_hash).await { - tracing::warn!( - "Failed to increment blob ref for copy {}: {}", - &blob_hash[..12], - e - ); - } + // No `add_reference` here: `copy_file_satellites` took it inside the + // transaction above, so a copy that exists always holds a reference. tracing::info!( "📋 BLOB COPY: {} (hash: {}, zero-copy via dedup)", @@ -742,6 +782,11 @@ impl FileWritePort for FileBlobWriteRepository { new_name: &str, caller_id: Uuid, ) -> Result { + // NFC-normalize the client-supplied name at the last touch — same + // reasoning as `save_file_with_blob_impl`. REST rename, WebDAV + // MOVE-with-rename, NC MOVE-with-rename all funnel here. + let new_name = normalize_storage_name(new_name); + // §14: `updated_by = $3` (caller_id), see move_file. let row = sqlx::query_as::< _, @@ -767,7 +812,7 @@ impl FileWritePort for FileBlobWriteRepository { created_by, updated_by "#, ) - .bind(new_name) + .bind(&new_name) .bind(file_id) .bind(caller_id) .fetch_optional(self.pool.as_ref()) @@ -855,6 +900,14 @@ impl FileWritePort for FileBlobWriteRepository { size: u64, caller_id: Uuid, ) -> Result<(File, PathBuf), DomainError> { + // NFC-normalize at the last touch before the DB bind — same + // reasoning as `save_file_with_blob_impl`. Deferred registration + // is the write-behind cache's fast-path (row up first, blob + // hash filled in on the async callback); it takes fresh client + // input via chunked-upload finalize among others, so NFD is + // reachable here too. + let name = normalize_storage_name_owned(name); + // For deferred registration we use a placeholder hash. // The write-behind cache will call update_file_content later. let placeholder_hash = "0000000000000000000000000000000000000000000000000000000000000000"; @@ -1024,10 +1077,21 @@ impl FileWritePort for FileBlobWriteRepository { DomainError::internal_error("FileBlobWrite", format!("fetch blob_hash: {e}")) })?; - // DELETE fires trg_files_decrement_blob_ref → storage.blobs.ref_count-- + // DELETE fires `trg_files_decrement_blob_ref` — post-2026-08-23 + // it dispatches manifest-first (see migration + // `20261017000000_file_delete_trigger_manifest_aware.sql`): + // decrements `chunk_manifests.ref_count` if the hash names a + // manifest (walking chunks on last-ref), else falls back to + // `storage.blobs.ref_count`. Counter state after this call is + // already correct. self.delete_file(file_id).await?; - // If the blob is now unreferenced, remove disk file + thumbnails. + // Physical cleanup only. `cleanup_if_orphaned` was previously + // manifest-aware and did counter compensation for the old + // trigger's over-decrement; after the trigger rewrite it's a + // legacy-blob-eager-reap helper — safe to keep calling + // unconditionally (no-op for CDC hashes; reaps legacy blobs + // that reached ref_count = 0). if let Some(hash) = blob_hash { self.dedup.cleanup_if_orphaned(&hash).await; } @@ -1041,6 +1105,13 @@ impl FileWritePort for FileBlobWriteRepository { target_parent_id: Option, dest_name: Option, ) -> Result { + // NFC-normalize the caller-supplied rename before handing off to + // the PG stored function. `dest_name = None` keeps the source's + // stored name (already normalized on ingest for post-fix rows; + // pre-fix historical NFD deliberately preserved). Only WebDAV + // COPY-a-folder-tree-with-rename passes a fresh client string. + let dest_name = dest_name.map(normalize_storage_name_owned); + let row = sqlx::query_as::<_, (String, i64, i64)>( "SELECT new_root_id, folders_copied, files_copied \ FROM storage.copy_folder_tree($1::uuid, $2::uuid, $3)", diff --git a/src/infrastructure/repositories/pg/folder_db_repository.rs b/src/infrastructure/repositories/pg/folder_db_repository.rs index d407ea05..e6ce6167 100644 --- a/src/infrastructure/repositories/pg/folder_db_repository.rs +++ b/src/infrastructure/repositories/pg/folder_db_repository.rs @@ -18,7 +18,7 @@ use crate::common::errors::DomainError; use crate::domain::entities::folder::Folder; use crate::domain::repositories::folder_repository::FolderRepository; use crate::domain::services::authorization::ResourceKind; -use crate::domain::services::path_service::StoragePath; +use crate::domain::services::path_service::{StoragePath, normalize_storage_name_owned}; /// Type alias for folder metadata rows from SQL queries. /// Tuple order: id, name, path, parent_id, drive_id, created_at, @@ -261,6 +261,21 @@ impl FolderRepository for FolderDbRepository { parent_id: Option, caller_id: Uuid, ) -> Result { + // Belt-and-suspenders NFC normalization at the last touch before the + // DB write. Every caller that lands here — REST `POST /api/folders`, + // WebDAV `MKCOL`, NextCloud `MKCOL`, batch create, WebDAV/NC `COPY` + // fall-through — passes the raw client-supplied name. macOS Finder / + // Android sync clients emit NFD path segments; if we bind them raw + // the DB row's bytes don't match NFC-normalizing clients' subsequent + // PROPFINDs (see AtalayaLabs/OxiCloud#706). Canonicalising here is + // the single choke-point that closes every entry-point audited on + // 2026-09-03 without asking each handler to remember. Fast-path + // `is_nfc_quick` inside `normalize_storage_name_owned` returns the + // owned string unchanged for names already in NFC — every visible + // ASCII name, every browser-composed non-ASCII name — so this costs + // one table-lookup on the hot path. + let name = normalize_storage_name_owned(name); + // Derive `drive_id` from the parent folder. Root-level folders // are reserved for the atomic drive-creation transaction in // `DrivePgRepository::create_personal_drive_atomic` (see @@ -689,6 +704,11 @@ impl FolderRepository for FolderDbRepository { new_name: String, caller_id: Uuid, ) -> Result { + // NFC-normalize the client-supplied new name — same reasoning as + // `create_folder` above (WebDAV `MOVE`, NC `MOVE`, REST rename all + // funnel here with raw client bytes). Cheap on the common path. + let new_name = normalize_storage_name_owned(new_name); + // The BEFORE UPDATE trigger recomputes path/lpath for this row; // the AFTER UPDATE cascade trigger then batch-updates all // descendants in a single UPDATE using the GiST lpath index. diff --git a/src/infrastructure/repositories/pg/mod.rs b/src/infrastructure/repositories/pg/mod.rs index 313e183c..99483610 100644 --- a/src/infrastructure/repositories/pg/mod.rs +++ b/src/infrastructure/repositories/pg/mod.rs @@ -1,5 +1,6 @@ mod address_book_pg_repository; mod app_password_pg_repository; +pub mod blob_reference_sources; mod calendar_event_pg_repository; mod calendar_pg_repository; mod contact_group_pg_repository; diff --git a/src/infrastructure/repositories/pg/session_pg_repository.rs b/src/infrastructure/repositories/pg/session_pg_repository.rs index b4c638be..336b8e80 100644 --- a/src/infrastructure/repositories/pg/session_pg_repository.rs +++ b/src/infrastructure/repositories/pg/session_pg_repository.rs @@ -117,7 +117,7 @@ impl SessionRepository for SessionPgRepository { SELECT id, user_id, refresh_token, expires_at, ip_address, user_agent, created_at, revoked, family_id, - oidc_id_token, oidc_sid, dpop_jkt, origin + oidc_id_token, oidc_sid, dpop_jkt, origin, last_seen_at FROM auth.sessions WHERE id = $1 "#, @@ -141,6 +141,7 @@ impl SessionRepository for SessionPgRepository { row.get("oidc_sid"), row.get("dpop_jkt"), crate::domain::entities::session::SessionOrigin::from_wire(row.get("origin")), + row.get("last_seen_at"), )) } @@ -155,7 +156,7 @@ impl SessionRepository for SessionPgRepository { SELECT id, user_id, refresh_token, expires_at, ip_address, user_agent, created_at, revoked, family_id, - oidc_id_token, oidc_sid, dpop_jkt, origin + oidc_id_token, oidc_sid, dpop_jkt, origin, last_seen_at FROM auth.sessions WHERE refresh_token = $1 "#, @@ -179,6 +180,7 @@ impl SessionRepository for SessionPgRepository { row.get("oidc_sid"), row.get("dpop_jkt"), crate::domain::entities::session::SessionOrigin::from_wire(row.get("origin")), + row.get("last_seen_at"), )) } @@ -192,7 +194,7 @@ impl SessionRepository for SessionPgRepository { SELECT id, user_id, refresh_token, expires_at, ip_address, user_agent, created_at, revoked, family_id, - oidc_id_token, oidc_sid, dpop_jkt, origin + oidc_id_token, oidc_sid, dpop_jkt, origin, last_seen_at FROM auth.sessions WHERE user_id = $1 ORDER BY created_at DESC @@ -220,6 +222,7 @@ impl SessionRepository for SessionPgRepository { row.get("oidc_sid"), row.get("dpop_jkt"), crate::domain::entities::session::SessionOrigin::from_wire(row.get("origin")), + row.get("last_seen_at"), ) }) .collect(); @@ -248,7 +251,7 @@ impl SessionRepository for SessionPgRepository { SELECT id, user_id, refresh_token, expires_at, ip_address, user_agent, created_at, revoked, family_id, - oidc_id_token, oidc_sid, dpop_jkt, origin + oidc_id_token, oidc_sid, dpop_jkt, origin, last_seen_at FROM auth.sessions WHERE ($1::uuid IS NULL OR user_id = $1) AND ($2 OR (revoked = false AND expires_at > NOW())) @@ -281,6 +284,7 @@ impl SessionRepository for SessionPgRepository { row.get("oidc_sid"), row.get("dpop_jkt"), crate::domain::entities::session::SessionOrigin::from_wire(row.get("origin")), + row.get("last_seen_at"), ) }) .collect(); diff --git a/src/infrastructure/repositories/pg/user_pg_repository.rs b/src/infrastructure/repositories/pg/user_pg_repository.rs index 6feb033e..7dd3bae0 100644 --- a/src/infrastructure/repositories/pg/user_pg_repository.rs +++ b/src/infrastructure/repositories/pg/user_pg_repository.rs @@ -7,7 +7,7 @@ use crate::application::ports::auth_ports::UserStoragePort; use crate::common::errors::DomainError; use crate::domain::entities::user::{User, UserFlags, UserRole}; use crate::domain::repositories::user_repository::{ - StorageStats, UserListEntry, UserRepository, UserRepositoryError, UserRepositoryResult, + StorageStats, UserRepository, UserRepositoryError, UserRepositoryResult, }; use crate::infrastructure::repositories::pg::transaction_utils::with_transaction; @@ -388,6 +388,90 @@ impl UserRepository for UserPgRepository { )) } + async fn get_user_with_derived_flags( + &self, + id: Uuid, + ) -> UserRepositoryResult<( + User, + crate::domain::repositories::user_repository::UserDerivedFlags, + )> { + // Same column set as `get_user_by_id` plus the three IS-NOT-NULL + // derivations for auth-capability flags AND the EXISTS scalar + // for `is_online`. The `interval` argument is bound as `$2` + // (seconds, `ONLINE_WINDOW.as_secs_f64()`) via + // `make_interval(secs => $2)` — same pattern as + // `session_liveness_gauges.rs`. Partial index + // `idx_sessions_last_seen_at WHERE revoked = FALSE` covers the + // EXISTS scan, so per-row cost is ~μs. + let row = sqlx::query( + r#" + SELECT + id, username, email, password_hash, role::text as role_text, + storage_quota_bytes, storage_used_bytes, + created_at, updated_at, last_login_at, active, + federation_kind, federation_issuer, federation_subject, image, is_external, + given_name, family_name, email_verified_at, preferred_locale, notify_on_share, + ui_preferences, + (password_hash IS NOT NULL) AS has_password, + (opaque_envelope IS NOT NULL) AS opaque_registered, + (opaque_migrated_at IS NOT NULL) AS opaque_migrated, + EXISTS ( + SELECT 1 FROM auth.sessions s + WHERE s.user_id = auth.users.id + AND s.revoked = FALSE + AND s.last_seen_at > NOW() - make_interval(secs => $2) + ) AS is_online + FROM auth.users + WHERE id = $1 + "#, + ) + .bind(id) + .bind(crate::application::dtos::session_dto::ONLINE_WINDOW.as_secs_f64()) + .fetch_one(&*self.pool) + .await + .map_err(Self::map_sqlx_error)?; + + let role_str: Option = row.try_get("role_text").unwrap_or(None); + let role = match role_str.as_deref() { + Some("admin") => UserRole::Admin, + _ => UserRole::User, + }; + + let user = User::from_data_full( + row.get("id"), + row.get("username"), + row.get("email"), + row.get("password_hash"), + role, + row.get("storage_quota_bytes"), + row.get("storage_used_bytes"), + row.get("created_at"), + row.get("updated_at"), + row.get("last_login_at"), + row.get("active"), + row.get::, _>("federation_kind") + .as_deref() + .and_then(crate::domain::entities::user::FederationKind::parse), + row.get("federation_issuer"), + row.get("federation_subject"), + row.get("image"), + row.get("is_external"), + row.get("given_name"), + row.get("family_name"), + row.get("email_verified_at"), + row.get("preferred_locale"), + row.get("notify_on_share"), + row.get::("ui_preferences"), + ); + let flags = crate::domain::repositories::user_repository::UserDerivedFlags { + has_password: row.get("has_password"), + opaque_registered: row.get("opaque_registered"), + opaque_migrated: row.get("opaque_migrated"), + is_online: row.get("is_online"), + }; + Ok((user, flags)) + } + /// Gets a user by username async fn get_user_by_username(&self, username: &str) -> UserRepositoryResult { let row = sqlx::query( @@ -835,103 +919,107 @@ impl UserRepository for UserPgRepository { Ok(users) } - async fn list_user_summaries( + async fn list_users_with_derived_flags( &self, limit: i64, offset: i64, include_external: bool, - ) -> UserRepositoryResult> { - let rows = sqlx::query_as::< - _, - ( - Uuid, - Option, - String, - String, - i64, - i64, - Option>, - bool, - Option, - Option, - bool, - bool, - bool, - bool, - ), - >( - // Auth-credential columns projected as booleans via `IS NOT - // NULL` rather than as timestamps / hashes so the row-mapping - // tuple stays small and the wire shape is exactly what the - // admin table needs. Per-row scalar tests — no cost beyond - // the full-table sequential scan the LIMIT/OFFSET already - // pays. `has_password` on the password_hash column tells - // the admin table whether a server-verifiable password is - // on file; combined with the two OPAQUE flags and - // federation_kind / federation_issuer, the SPA derives the - // full "capability set" per user (password / OPAQUE / SSO / - // passwordless). + ) -> UserRepositoryResult< + Vec<( + User, + crate::domain::repositories::user_repository::UserDerivedFlags, + )>, + > { + // Full `User` column set (matches `get_user_by_id`) + the four + // derived booleans (IS-NOT-NULL for auth capability, EXISTS for + // `is_online`) in one SELECT. Same rationale as the single-user + // `get_user_with_derived_flags` variant. Widened over the older + // `list_user_summaries` projection because the FE now consumes + // the full user profile from these rows (killing the per-row + // `/api/users/{id}` fetch the admin table used to fire for + // avatars — see docs/plan/userdto-refactor.md § N+1). + // + // `interval` bound as `$4` seconds + // (`ONLINE_WINDOW.as_secs_f64()`), same pattern as + // `session_liveness_gauges.rs` and `get_user_with_derived_flags`. + let rows = sqlx::query( r#" SELECT - id, username, email, role::text, + id, username, email, password_hash, role::text as role_text, storage_quota_bytes, storage_used_bytes, - last_login_at, active, - federation_kind, federation_issuer, is_external, - (password_hash IS NOT NULL) AS has_password, + created_at, updated_at, last_login_at, active, + federation_kind, federation_issuer, federation_subject, image, is_external, + given_name, family_name, email_verified_at, preferred_locale, notify_on_share, + ui_preferences, + (password_hash IS NOT NULL) AS has_password_flag, (opaque_envelope IS NOT NULL) AS opaque_registered, - (opaque_migrated_at IS NOT NULL) AS opaque_migrated - FROM auth.users - WHERE ($3 OR is_external = FALSE) - ORDER BY created_at DESC, id DESC - LIMIT $1 OFFSET $2 + (opaque_migrated_at IS NOT NULL) AS opaque_migrated, + EXISTS ( + SELECT 1 FROM auth.sessions s + WHERE s.user_id = auth.users.id + AND s.revoked = FALSE + AND s.last_seen_at > NOW() - make_interval(secs => $4) + ) AS is_online + FROM auth.users + WHERE ($3 OR is_external = FALSE) + ORDER BY created_at DESC, id DESC + LIMIT $1 OFFSET $2 "#, ) .bind(limit) .bind(offset) .bind(include_external) - .fetch_all(self.pool.as_ref()) + .bind(crate::application::dtos::session_dto::ONLINE_WINDOW.as_secs_f64()) + .fetch_all(&*self.pool) .await .map_err(Self::map_sqlx_error)?; + // Note: the `has_password_flag` alias avoids colliding with the + // `password_hash` column selected above (the tuple destructure + // in `list_user_summaries` uses a shorter projection so it + // could reuse the raw `has_password` alias; here we keep both). Ok(rows .into_iter() - .map( - |( - id, - username, - email, + .map(|row| { + let role_str: Option = row.try_get("role_text").unwrap_or(None); + let role = match role_str.as_deref() { + Some("admin") => UserRole::Admin, + _ => UserRole::User, + }; + let user = User::from_data_full( + row.get("id"), + row.get("username"), + row.get("email"), + row.get("password_hash"), role, - storage_quota_bytes, - storage_used_bytes, - last_login_at, - active, - federation_kind, - federation_issuer, - is_external, - has_password, - opaque_registered, - opaque_migrated, - )| UserListEntry { - id, - username, - email, - role: if role == "admin" { - UserRole::Admin - } else { - UserRole::User - }, - storage_quota_bytes, - storage_used_bytes, - last_login_at, - active, - federation_kind, - federation_issuer, - is_external, - has_password, - opaque_registered, - opaque_migrated, - }, - ) + row.get("storage_quota_bytes"), + row.get("storage_used_bytes"), + row.get("created_at"), + row.get("updated_at"), + row.get("last_login_at"), + row.get("active"), + row.get::, _>("federation_kind") + .as_deref() + .and_then(crate::domain::entities::user::FederationKind::parse), + row.get("federation_issuer"), + row.get("federation_subject"), + row.get("image"), + row.get("is_external"), + row.get("given_name"), + row.get("family_name"), + row.get("email_verified_at"), + row.get("preferred_locale"), + row.get("notify_on_share"), + row.get::("ui_preferences"), + ); + let flags = crate::domain::repositories::user_repository::UserDerivedFlags { + has_password: row.get("has_password_flag"), + opaque_registered: row.get("opaque_registered"), + opaque_migrated: row.get("opaque_migrated"), + is_online: row.get("is_online"), + }; + (user, flags) + }) .collect()) } @@ -1302,6 +1390,21 @@ impl UserStoragePort for UserPgRepository { .map_err(DomainError::from) } + async fn get_user_with_derived_flags( + &self, + id: Uuid, + ) -> Result< + ( + User, + crate::domain::repositories::user_repository::UserDerivedFlags, + ), + DomainError, + > { + UserRepository::get_user_with_derived_flags(self, id) + .await + .map_err(DomainError::from) + } + async fn get_users_by_ids(&self, ids: Vec) -> Result, DomainError> { UserRepository::get_users_by_ids(self, ids) .await @@ -1356,13 +1459,19 @@ impl UserStoragePort for UserPgRepository { .map_err(DomainError::from) } - async fn list_user_summaries( + async fn list_users_with_derived_flags( &self, limit: i64, offset: i64, include_external: bool, - ) -> Result, DomainError> { - UserRepository::list_user_summaries(self, limit, offset, include_external) + ) -> Result< + Vec<( + User, + crate::domain::repositories::user_repository::UserDerivedFlags, + )>, + DomainError, + > { + UserRepository::list_users_with_derived_flags(self, limit, offset, include_external) .await .map_err(DomainError::from) } @@ -1743,26 +1852,27 @@ mod integration_tests { ) .await; - let page = UserRepository::list_user_summaries(&repo, 3, 0, true) + // Migrated from the (now-deleted) `list_user_summaries` + + // `UserListEntry` to `list_users_with_derived_flags`, which + // returns `Vec<(User, UserDerivedFlags)>`. Field checks read + // through the `User` accessors instead of struct-field access. + let page = UserRepository::list_users_with_derived_flags(&repo, 3, 0, true) .await .expect("compact projection query must decode"); - assert_eq!(page.iter().map(|entry| entry.id).collect::>(), ids); - assert_eq!(page[0].username.as_deref(), Some(username_a.as_str())); - assert_eq!(page[0].role, UserRole::Admin); - assert_eq!(page[0].storage_quota_bytes, 10_737_418_240); - assert_eq!(page[1].username, None); - assert!(page[1].is_external); - assert_eq!( - page[1].federation_issuer.as_deref(), - Some("integration-idp") - ); + assert_eq!(page.iter().map(|(u, _)| u.id()).collect::>(), ids); + assert_eq!(page[0].0.username(), Some(username_a.as_str())); + assert_eq!(page[0].0.role(), UserRole::Admin); + assert_eq!(page[0].0.storage_quota_bytes(), 10_737_418_240); + assert_eq!(page[1].0.username(), None); + assert!(page[1].0.is_external()); + assert_eq!(page[1].0.federation_issuer(), Some("integration-idp")); - let internal = UserRepository::list_user_summaries(&repo, 10, 0, false) + let internal = UserRepository::list_users_with_derived_flags(&repo, 10, 0, false) .await .expect("internal compact projection query must decode"); - assert!(internal.iter().any(|entry| entry.id == ids[0])); - assert!(internal.iter().any(|entry| entry.id == ids[2])); - assert!(!internal.iter().any(|entry| entry.id == ids[1])); + assert!(internal.iter().any(|(u, _)| u.id() == ids[0])); + assert!(internal.iter().any(|(u, _)| u.id() == ids[2])); + assert!(!internal.iter().any(|(u, _)| u.id() == ids[1])); sqlx::query("DELETE FROM auth.users WHERE id = ANY($1)") .bind(ids.as_slice()) diff --git a/src/infrastructure/scheduler/handler.rs b/src/infrastructure/scheduler/handler.rs index 011bfa30..8135db26 100644 --- a/src/infrastructure/scheduler/handler.rs +++ b/src/infrastructure/scheduler/handler.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; -use super::types::{JobOutcome, JobRunArgs}; +use super::types::{JobOutcome, JobRunArgs, Mutates}; /// Implemented by every service that wants to run on a fixed interval /// through the periodic scheduler. @@ -92,4 +92,40 @@ pub trait JobHandler: Send + Sync { fn is_recoverable(&self) -> bool { false } + + /// What this job does, in one or two sentences, for the admin UI. + /// + /// English, in the trait, beside the behaviour it describes — not in + /// `locales/*.json`. A description that lives away from the code rots + /// the moment a job changes, invisibly, and a translator cannot know + /// what `manifests_consistency` reconciles. i18n can layer on later + /// keyed by job name with this as the fallback, so a missing + /// translation degrades to English rather than to a blank panel. + /// + /// Defaulted to `""` so adding it to the existing jobs is incremental + /// rather than one breaking change; the UI omits the line when empty. + fn description(&self) -> &'static str { + "" + } + + /// Whether a run changes state, and under what conditions. See + /// [`Mutates`] for why this is not a boolean. + fn mutates(&self) -> Mutates { + Mutates::Never + } + + /// `Some(..)` when `?repair=true` does something beyond a default run, + /// describing what it ADDS; `None` when the flag is inert. + /// + /// One method rather than a `supports_repair` boolean plus prose: its + /// presence drives whether the UI offers the toggle, its content drives + /// the confirmation text. A boolean would leave the frontend to invent + /// wording for a destructive action it does not understand. + /// + /// Independent of [`Self::mutates`], not derived from it — the thumbnail + /// import jobs are [`Mutates::Always`] *and* repair-capable, inserting + /// rows on a plain run and additionally unlinking sidecars under repair. + fn repair_description(&self) -> Option<&'static str> { + None + } } diff --git a/src/infrastructure/scheduler/mod.rs b/src/infrastructure/scheduler/mod.rs index cb2e50c5..7e16b270 100644 --- a/src/infrastructure/scheduler/mod.rs +++ b/src/infrastructure/scheduler/mod.rs @@ -37,5 +37,7 @@ pub use recoverable::{ RecoverableJobHandler, RunOutcome, RunProgress, RunStatus, RunSummary, derive_progress, record_or_log, run_or_resume, }; -pub use registry::{JobEntry, JobRegistry, JobSummary, PausedRunBrief, RegisterError}; -pub use types::{ErrCause, JobOutcome, JobRunArgs}; +pub use registry::{ + JobEntry, JobRegistry, JobSummary, PausedRunBrief, RegisterError, StartupTrigger, +}; +pub use types::{ErrCause, JobOutcome, JobRunArgs, Mutates}; diff --git a/src/infrastructure/scheduler/recoverable.rs b/src/infrastructure/scheduler/recoverable.rs index fa5d468d..99df9b55 100644 --- a/src/infrastructure/scheduler/recoverable.rs +++ b/src/infrastructure/scheduler/recoverable.rs @@ -48,7 +48,7 @@ use uuid::Uuid; use crate::common::errors::DomainError; use super::handler::JobHandler; -use super::types::{JobOutcome, JobRunArgs}; +use super::types::{JobOutcome, JobRunArgs, Mutates}; // ─── Run status ───────────────────────────────────────────────────────────── @@ -199,6 +199,63 @@ impl RunOutcome { } } +/// Write `JobRunArgs` to `params` on a Fresh run, or read them back on a +/// Resumed one. +/// +/// Returns the args the handler should actually use. On resume that is +/// whatever the original run recorded, NOT what the resuming caller +/// passed — see the call site in [`run_or_resume`] for why changing mode +/// mid-run is refused. +/// +/// Every flag is stored as a string, matching the `params` convention the +/// progress fields already use, and each is read back independently: a run +/// paused before this existed simply has no keys, and each missing one +/// falls back to `false` / `None`. That is the safe direction — a resumed +/// legacy run under-acts rather than deleting under a flag nobody gave it. +async fn persist_or_restore_args( + store: &dyn JobStore, + args: &JobRunArgs, + is_fresh: bool, +) -> Result { + const FLAGS: [&str; 3] = ["force", "deep", "repair"]; + + if is_fresh { + for (key, value) in FLAGS.iter().zip([args.force, args.deep, args.repair]) { + let v = if value { "true" } else { "false" }; + store + .set_string_param(key, v) + .await + .map_err(|e| format!("persist `{key}` to params: {e}"))?; + } + // `storage` is absent rather than empty when unset, so a run that + // did not scope itself does not grow a key claiming it did. + if let Some(name) = &args.storage { + store + .set_string_param("storage", name) + .await + .map_err(|e| format!("persist `storage` to params: {e}"))?; + } + return Ok(args.clone()); + } + + let mut restored = JobRunArgs::default(); + for (key, slot) in FLAGS.iter().zip([ + &mut restored.force, + &mut restored.deep, + &mut restored.repair, + ]) { + *slot = match store.get_string_param(key).await { + Ok(v) => v.as_deref() == Some("true"), + Err(e) => return Err(format!("read `{key}` from params: {e}")), + }; + } + restored.storage = store + .get_string_param("storage") + .await + .map_err(|e| format!("read `storage` from params: {e}"))?; + Ok(restored) +} + // ─── Traits — implementor + port ──────────────────────────────────────────── /// The implementor-facing contract for a long-running, restart-tolerant @@ -238,6 +295,47 @@ pub trait RecoverableJobHandler: Send + Sync { /// URL fragment: `POST /api/admin/jobs/{name}/trigger`. fn name(&self) -> &str; + /// What this job does, for the admin UI. + /// + /// English, in the trait, beside the behaviour it describes — not in + /// `locales/*.json`. A description that lives away from the code rots the + /// moment a job changes, invisibly, and a translator cannot know what + /// `manifests_consistency` reconciles. i18n can layer on later keyed by + /// job name, with this as the fallback, so a missing translation degrades + /// to English rather than a blank panel. + /// + /// Defaulted so adding it to ~15 existing jobs is incremental rather than + /// one breaking change. + fn description(&self) -> &'static str { + "" + } + + /// Whether a run changes state, and under what conditions. + /// + /// Three values rather than a boolean because there are three cases, and + /// the interesting one is conditional: a job can be read-only by default + /// and destructive under `?repair=true`. A boolean forces that job to + /// answer wrongly for one of its two modes — `false` on something that + /// can delete files is actively misleading. + fn mutates(&self) -> Mutates { + Mutates::Never + } + + /// `Some(..)` when `?repair=true` does something beyond a default run, + /// describing what it ADDS; `None` when the flag is inert. + /// + /// One method rather than a `supports_repair` boolean plus prose: its + /// presence drives whether the UI offers the toggle, its content drives + /// the confirmation text. A boolean would leave the frontend to invent + /// wording for a destructive action it does not understand. + /// + /// Independent of [`Self::mutates`], not derived from it — the import + /// jobs are [`Mutates::Always`] *and* repair-capable, inserting rows on a + /// plain run and additionally unlinking files under repair. + fn repair_description(&self) -> Option<&'static str> { + None + } + /// Long-running scan. See trait-level doc for the contract. /// /// `store` — bound to THIS run (a single row in @@ -361,7 +459,15 @@ pub trait JobStore: Send + Sync { /// `"stale_used_bytes"`, `"missing_blob"`). Never rename across /// releases; new failure modes get new values. /// - /// `severity` — one of `"data_loss"`, `"inconsistent"`, `"anomaly"`. + /// `severity` — one of: + /// - `"data_loss"` — bytes / rows unreachable or gone. + /// - `"inconsistent"` — counters or materialised values wrong, + /// content intact. + /// - `"anomaly"` — surprising state worth surfacing, no known impact. + /// This is the level the admin panel labels "notices"; there is no + /// separate `notice` severity, and a job that acted on what it found + /// says so in `detail` rather than in a fourth severity that would + /// render identically. /// /// `resource_id` — the file / folder / drive / blob the finding /// pertains to. `None` for run-wide findings (e.g. "backend @@ -748,10 +854,44 @@ pub async fn run_or_resume( } } + // Bind the run to the flags it started with. + // + // A Fresh run records its `JobRunArgs` in `params`; a Resumed run reads + // them back and runs with THOSE, ignoring whatever the resuming caller + // passed. Two reasons, and the engine is the only place both are + // guaranteed: + // + // **A resumed run must not change mode.** Handlers read `args` on every + // call, so a paused `?repair=true` import resumed by a plain trigger + // silently continued as import-only — the deletion half never finished + // and nothing said so. The same held for `?deep=true`: a paused bit-rot + // scan resumed shallow while still reporting as the run that started + // deep. Fixing it per-handler meant every job remembering, and three of + // them did not. + // + // **The run row should say what it did.** For a destructive job, "did + // this run delete anything?" is answerable only from `params`, and that + // is what an operator reads afterwards. + // + // Deliberately NOT overridable on resume. Adding `?repair=true` to a + // resume would apply it to the remaining entries only, producing a run + // that half-deleted — the honest way to change your mind is to cancel + // and start fresh. + let args = match persist_or_restore_args(&*store, args, is_fresh).await { + Ok(effective) => effective, + Err(e) => { + // Fail the run rather than guess. Proceeding would mean acting + // under flags nothing recorded, which for the jobs that delete + // is the one thing worth refusing. + log_terminal_write_err("mark_failed", run_id, store.mark_failed(&e).await); + return JobOutcome::err(e); + } + }; + // Dispatch. Terminal writes to `jobs.recoverable_runs` happen // here (NOT in the handler) so the row always ends in a state // that matches what the handler returned. - let outcome = job.run_resumable(&*store, args, resume_cursor).await; + let outcome = job.run_resumable(&*store, &args, resume_cursor).await; // Fetch the terminal run summary so we can surface aggregate // stats (finding_count, scanned_count) on the outer JobOutcome @@ -993,6 +1133,21 @@ impl JobHandler for RecoverableAdapter { // downstream. true } + + // The registry only ever sees `dyn JobHandler`, so the tenant's own + // metadata has to be forwarded through the wrapper or it is invisible + // to `GET /api/admin/jobs`. Silently returning the JobHandler defaults + // here would leave every recoverable job undescribed and reported as + // read-only — including ones that delete files. + fn description(&self) -> &'static str { + self.inner.description() + } + fn mutates(&self) -> Mutates { + self.inner.mutates() + } + fn repair_description(&self) -> Option<&'static str> { + self.inner.repair_description() + } } // ─── Ergonomics: JobRegistry extension for recoverable jobs ───────────────── @@ -1504,6 +1659,47 @@ mod tests { // ─── Tests ───────────────────────────────────────────────────────────── + /// The registry only ever sees `dyn JobHandler`, so a recoverable + /// tenant's metadata reaches `GET /api/admin/jobs` only if the adapter + /// forwards it. Falling back to the `JobHandler` defaults here would + /// report every recoverable job as undescribed and read-only — + /// including the imports, which delete files under repair. + #[tokio::test] + async fn adapter_forwards_job_metadata_from_inner_handler() { + struct Annotated; + #[async_trait] + impl RecoverableJobHandler for Annotated { + fn name(&self) -> &str { + "annotated" + } + async fn run_resumable( + &self, + _store: &dyn JobStore, + _args: &JobRunArgs, + _resume_cursor: Option>, + ) -> RunOutcome { + RunOutcome::completed() + } + fn description(&self) -> &'static str { + "walks a thing" + } + fn mutates(&self) -> Mutates { + Mutates::OnRepairOnly + } + fn repair_description(&self) -> Option<&'static str> { + Some("fixes the thing") + } + } + + let provider: Arc = Arc::new(MemProvider::new()); + let adapter = RecoverableAdapter::new(Arc::new(Annotated), provider); + let as_handler: &dyn JobHandler = &adapter; + + assert_eq!(as_handler.description(), "walks a thing"); + assert_eq!(as_handler.mutates(), Mutates::OnRepairOnly); + assert_eq!(as_handler.repair_description(), Some("fixes the thing")); + } + #[tokio::test] async fn fresh_run_completes_and_marks_status_completed() { let provider = Arc::new(MemProvider::new()); diff --git a/src/infrastructure/scheduler/registry.rs b/src/infrastructure/scheduler/registry.rs index ba112ce9..e698261c 100644 --- a/src/infrastructure/scheduler/registry.rs +++ b/src/infrastructure/scheduler/registry.rs @@ -20,7 +20,7 @@ use serde::Serialize; use tokio::sync::{RwLock, Semaphore}; use super::handler::JobHandler; -use super::types::{JobOutcome, JobRunArgs}; +use super::types::{JobOutcome, JobRunArgs, Mutates}; /// A registered job plus its runtime state. Held as `Arc` /// inside the registry so the engine can hold a snapshot across an @@ -135,6 +135,13 @@ impl JobRegistry { timeout: Option, ) -> Result<(), RegisterError> { let name = handler.name().to_string(); + // A job declaring it mutates only under a flag it does not support + // is self-contradictory, and the UI would render it as safe with no + // way to reach the mutating path. Cheap to catch here, invisible + // otherwise. + if handler.mutates() == Mutates::OnRepairOnly && handler.repair_description().is_none() { + return Err(RegisterError::RepairOnlyWithoutRepair(name)); + } let mut guard = self.entries.write().await; if guard.contains_key(&name) { return Err(RegisterError::DuplicateName(name)); @@ -220,17 +227,21 @@ impl JobRegistry { }; JobSummary { name, + description: entry.handler.description(), + mutates: entry.handler.mutates(), + repair_description: entry.handler.repair_description(), interval_ms: entry.interval.map(|d| d.as_millis() as u64), next_run_at: state.next_run_at, last_run_at, last_outcome, running: state.current_run_start.is_some(), recoverable: entry.handler.is_recoverable(), - // Populated in `list_jobs` handler via a single - // DB round-trip — kept out of the registry - // snapshot to avoid pulling a DB dependency into - // the in-memory scheduler state. + // Both populated in the `list_jobs` handler — one + // from a DB round-trip, one from AppConfig. Kept + // out of the registry snapshot so the in-memory + // scheduler state pulls in neither dependency. paused_run: None, + startup: None, } }) .collect() @@ -283,6 +294,11 @@ impl Default for JobRegistry { pub enum RegisterError { #[error("job name already registered: {0}")] DuplicateName(String), + #[error( + "job {0} declares mutates = OnRepairOnly but no repair_description() — \ + it claims to mutate only under a flag it does not support" + )] + RepairOnlyWithoutRepair(String), } /// Per-job row in the `GET /api/admin/jobs` response. @@ -304,6 +320,17 @@ pub enum RegisterError { #[derive(Debug, Clone, Serialize)] pub struct JobSummary { pub name: String, + /// One or two sentences on what the job does. Empty for jobs that + /// haven't declared one yet — the UI omits the line rather than + /// rendering a blank block. + #[serde(skip_serializing_if = "str::is_empty")] + pub description: &'static str, + pub mutates: Mutates, + /// `Some` iff the job does something extra under `?repair=true`. + /// Presence is what gates the repair toggle in the UI; the string + /// is the confirmation text. + #[serde(skip_serializing_if = "Option::is_none")] + pub repair_description: Option<&'static str>, #[serde(skip_serializing_if = "Option::is_none")] pub interval_ms: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -320,6 +347,32 @@ pub struct JobSummary { /// picks Resume when the latest row is Paused). #[serde(skip_serializing_if = "Option::is_none")] pub paused_run: Option, + /// Populated iff `OXICLOUD_STARTUP_JOBS` names this job — the flags + /// it will be dispatched with at every boot. + /// + /// Surfaced because the panel would otherwise be silently wrong + /// about the most consequential thing on the row: a job configured + /// with `repair=true` deletes files on every restart, and reading + /// the row you would think that only happens when someone clicks. + /// Filled by the `list_jobs` handler, which has the config; the + /// registry deliberately doesn't. + #[serde(skip_serializing_if = "Option::is_none")] + pub startup: Option, +} + +/// The flags a job configured in `OXICLOUD_STARTUP_JOBS` runs with. +/// +/// Mirrors `JobRunArgs` on the wire rather than embedding it, because +/// this is an API shape the admin panel switches on, and `JobRunArgs` +/// is an internal dispatch type free to change without a frontend +/// release. +#[derive(Debug, Clone, Serialize)] +pub struct StartupTrigger { + pub force: bool, + pub deep: bool, + pub repair: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub storage: Option, } /// Enough info about a paused recoverable run for the admin panel to @@ -393,6 +446,78 @@ mod tests { assert!(matches!(err, RegisterError::DuplicateName(_))); } + /// A job declaring `OnRepairOnly` without a `repair_description` has + /// no reachable mutating path — the UI gates the repair toggle on + /// that string's presence, so the job would render as safe and stay + /// read-only forever. Catch it at wiring time rather than let it read + /// as a working configuration. + #[tokio::test] + async fn repair_only_without_repair_description_rejected() { + struct Contradictory; + #[async_trait] + impl JobHandler for Contradictory { + fn name(&self) -> &str { + "contradictory" + } + async fn run(&self, _args: &JobRunArgs) -> JobOutcome { + JobOutcome::ok(0) + } + fn mutates(&self) -> Mutates { + Mutates::OnRepairOnly + } + // repair_description() left at its `None` default — the bug. + } + + let reg = JobRegistry::new(); + let err = reg + .try_register(Arc::new(Contradictory), None, None) + .await + .expect_err("OnRepairOnly without a repair_description must be rejected"); + assert!(matches!(err, RegisterError::RepairOnlyWithoutRepair(_))); + } + + /// The registry hands `dyn JobHandler` to the admin snapshot, so a + /// tenant's own metadata is only visible if it survives that erasure. + #[tokio::test] + async fn snapshot_carries_job_metadata() { + struct Described; + #[async_trait] + impl JobHandler for Described { + fn name(&self) -> &str { + "described" + } + async fn run(&self, _args: &JobRunArgs) -> JobOutcome { + JobOutcome::ok(0) + } + fn description(&self) -> &'static str { + "does a thing" + } + fn mutates(&self) -> Mutates { + Mutates::Always + } + fn repair_description(&self) -> Option<&'static str> { + Some("also deletes the thing") + } + } + + let reg = JobRegistry::new(); + reg.register(Arc::new(Described), None, None).await; + let snap = reg.snapshot().await; + let row = snap.iter().find(|j| j.name == "described").unwrap(); + assert_eq!(row.description, "does a thing"); + assert_eq!(row.mutates, Mutates::Always); + assert_eq!(row.repair_description, Some("also deletes the thing")); + + // Undeclared jobs stay at the safe defaults so the panel can tell + // "read-only" from "not yet described" — empty string, not prose. + reg.register(handler("bare"), None, None).await; + let snap = reg.snapshot().await; + let bare = snap.iter().find(|j| j.name == "bare").unwrap(); + assert_eq!(bare.description, ""); + assert_eq!(bare.mutates, Mutates::Never); + assert!(bare.repair_description.is_none()); + } + #[tokio::test] #[should_panic(expected = "DI wiring bug")] async fn register_panics_on_duplicate() { diff --git a/src/infrastructure/scheduler/types.rs b/src/infrastructure/scheduler/types.rs index 0cb22375..3a20378a 100644 --- a/src/infrastructure/scheduler/types.rs +++ b/src/infrastructure/scheduler/types.rs @@ -45,11 +45,25 @@ use serde::{Deserialize, Serialize}; /// of the entry to probe instead of the currently-active backend. /// `None` falls through to the live backend (today's behaviour). /// - Others — ignored. +/// +/// Semantics of `repair` (added 2026-10-17 for the refcount fix): +/// - `blobs_consistency` / `manifests_consistency` — when `true`, +/// after each `refcount_mismatch` / `manifest_refcount_mismatch` +/// finding is recorded, apply the corrective UPDATE that sets the +/// stored counter to the auditor's computed `actual_ref_count`. +/// Content-safe: the row itself is fine, only the counter is +/// wrong. Race-safe: each UPDATE recomputes the auditor formula +/// in the same statement, so a concurrent write can't leave a +/// stale value. Default `false` preserves discovery-only +/// behaviour. Also propagates through `consistency_batch` to +/// both tenants — one `?repair=true` call fixes both counters. +/// - Others — ignored. #[derive(Debug, Clone, Default)] pub struct JobRunArgs { pub force: bool, pub deep: bool, pub storage: Option, + pub repair: bool, } /// Uniform outcome the supervisor logs and stores for every job dispatch. @@ -153,10 +167,52 @@ impl fmt::Display for ErrCause { } } +/// When a job changes state. +/// +/// Drives how the admin UI presents a trigger: `Never` earns a read-only +/// badge, `OnRepairOnly` is safe to run and warns only when the toggle is on, +/// `Always` warns regardless. +/// +/// Three values rather than a boolean because there are three cases, and the +/// interesting one is conditional. `false` on a job that can delete files +/// under `?repair=true` is actively misleading; `true` on one that is +/// read-only by default is equally wrong. `OnRepairOnly` names the case a +/// boolean cannot, and it is where the recovery framework is heading — +/// discovery-only by default, mutation behind an explicit opt-in — so a +/// consistency tenant that later grows a repair arm changes this one value +/// and nothing else. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Mutates { + /// Read-only under every flag. All consistency tenants. + Never, + /// Changes state on a plain run. GC, janitors, the import jobs. + Always, + /// Read-only by default; mutates only under `?repair=true`. Pairing this + /// with `repair_description() == None` is contradictory — a job claiming + /// it mutates only under a flag it does not support. + OnRepairOnly, +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn mutates_serialises_snake_case() { + // The admin UI switches on these strings — a rename is a breaking + // change to the panel, not just to Rust callers. + assert_eq!(serde_json::to_string(&Mutates::Never).unwrap(), "\"never\""); + assert_eq!( + serde_json::to_string(&Mutates::Always).unwrap(), + "\"always\"" + ); + assert_eq!( + serde_json::to_string(&Mutates::OnRepairOnly).unwrap(), + "\"on_repair_only\"" + ); + } + #[test] fn joboutcome_kind_label() { assert_eq!(JobOutcome::ok(0).kind(), "ok"); diff --git a/src/infrastructure/services/azure_blob_backend.rs b/src/infrastructure/services/azure_blob_backend.rs index 622af000..e8a18b43 100644 --- a/src/infrastructure/services/azure_blob_backend.rs +++ b/src/infrastructure/services/azure_blob_backend.rs @@ -13,7 +13,8 @@ use futures::{StreamExt, TryStreamExt}; use tokio::fs; use crate::application::ports::blob_storage_ports::{ - BlobStorageBackend, BlobStream, StorageHealthStatus, + BackendBlobEntry, BackendUnknownEntry, BlobListPage, BlobStorageBackend, BlobStream, + StorageHealthStatus, }; use crate::common::config::AzureStorageConfig; use crate::domain::errors::{DomainError, ErrorKind}; @@ -55,6 +56,29 @@ impl AzureBlobBackend { } } + /// Inverse of [`Self::blob_name`] — the hash a listing entry names, + /// or `None` when the entry is not one of ours. + /// + /// Mirrors `S3BlobBackend::hash_from_object_key`, including the check + /// that the shard equals the hash's own first two characters: without + /// it, `blob_name(hash)` would not reproduce the name we just parsed, + /// and a mis-sharded object would be reported as a live blob that no + /// read path can find. + fn hash_from_blob_name(name: &str) -> Option { + let (prefix, rest) = name.split_once('/')?; + if prefix.len() != 2 || !prefix.chars().all(|c| c.is_ascii_hexdigit()) { + return None; + } + let stem = rest.strip_suffix(".blob")?; + if stem.len() != 64 || !stem.chars().all(|c| c.is_ascii_hexdigit()) { + return None; + } + if !stem.starts_with(prefix) { + return None; + } + Some(stem.to_string()) + } + /// Compute the blob name for a given hash. fn blob_name(hash: &str) -> String { let prefix = &hash[0..2]; @@ -243,6 +267,37 @@ impl BlobStorageBackend for AzureBlobBackend { }) } + /// # Known incompatibility: sub-4 MiB ranges break on Azurite + /// + /// `azure_core` 0.21's `Range::as_headers` + /// (`src/request_options/range.rs`) attaches + /// `x-ms-range-get-content-crc64: true` to **any range shorter than + /// 4 MiB**, unconditionally and with no opt-out. Real Azure honours + /// it; Azurite answers 500. `azure_core` then classifies 500 as + /// retryable and loops on a deterministic error, forever. + /// + /// The reachable path is `backend_migration` → + /// `EncryptedBlobBackend::head_check` → + /// `get_blob_range_stream(hash, 0, HEADER_SIZE)`. `HEADER_SIZE` is a + /// few dozen bytes, and it runs against the TARGET before each write, + /// so a local→Azurite migration hangs on its first blob while holding + /// `migration_readonly` — writes refused application-wide. + /// + /// **Deliberately not worked around here.** The available workaround + /// is to issue an unranged `get()` for small requests (its 16 MiB + /// `initial_range` clears the threshold, so the header is never sent) + /// and truncate client-side. That is correct against real Azure but + /// pays for an emulator with production cost: a ~40-byte format probe + /// becomes a whole-blob transfer, and it puts new offset arithmetic + /// on the read path, where a mistake serves wrong bytes silently + /// rather than failing. + /// + /// The real fix is the official `azure_storage_blob` 1.x, where + /// `range_get_content_crc64` is an explicit field on + /// `BlobClientDownloadOptions` — leave it unset and the request is + /// never made. Until then, the Azurite suite exercises enumeration + /// and round-trips but not migration; see + /// `tests/api/backend_consistency_azure.hurl`. fn get_blob_range_stream( &self, hash: &str, @@ -394,6 +449,197 @@ impl BlobStorageBackend for AzureBlobBackend { }) } + /// Enumerate blob hashes in lexicographic order, so + /// `backend_consistency` can merge-join against `storage.blobs` + /// instead of degrading to a per-row probe that structurally cannot + /// see orphans. + /// + /// ## Why this is a shard walk and not one flat listing + /// + /// **The cursor IS a blob hash**, not a provider token. The caller + /// forces that: it advances ONE cursor across both sides of the join, + /// feeding the same value here and to `WHERE hash > $1` in SQL. S3 + /// satisfies it with `start_after(object_key(cursor))`. + /// + /// Azure has no `StartAfter`. REST API 2023-05-03 added `startFrom`, + /// which would be the direct equivalent — but this SDK + /// (`azure_storage_blobs` 0.21, archived) never sends it: `ListBlobs` + /// exposes only `prefix`, `delimiter`, `max_results` and `marker`, + /// and `marker` is an opaque continuation token that cannot be + /// derived from a hash. + /// + /// So resume rides on `prefix` instead. Names are + /// `{hash[0..2]}/{hash}.blob`, which partitions the container into + /// 256 shards that are themselves in hash order. Walking + /// `00/` … `ff/` therefore yields exactly the global hash order, and + /// a cursor names the shard to restart in. Re-listing on resume is + /// bounded by shard width — 1/256th of the container — rather than + /// by the whole container, which is what a client-side skip over a + /// flat listing would cost on every single page. + /// + /// `marker` is used only INSIDE one call, to page within a shard, and + /// never escapes as the cursor — the same treatment the S3 impl gives + /// its continuation token. + /// + /// ## What this does NOT see, unlike S3 + /// + /// S3 lists the bucket with no prefix, so any foreign object lands in + /// `unknowns`. Constraining to `{2-hex}/` means foreign names outside + /// that shape are invisible here. + /// + /// That asymmetry is deliberate and safe in the direction that + /// matters: an orphan is a blob **we** wrote and later stopped + /// referencing, so it always has the canonical name and is always + /// enumerated. Only genuinely foreign files — another workload + /// sharing the container — can be missed, and they are informational + /// notices, never findings. Trading them for O(N) enumeration instead + /// of O(N²/limit) is worth it. + fn list_blob_hashes( + &self, + cursor: Option, + limit: usize, + ) -> Pin> + Send + '_>> + { + Box::pin(async move { + // A run of foreign entries can't produce a resume cursor, and + // buffering the container to find one blob is worse than + // failing. Mirrors the S3 impl's bound, and like it is on + // entries accumulated rather than requests made: request + // count scales with the caller's `limit`, so a request cap + // would fire on a healthy container merely because the caller + // paged finely. + const MAX_UNKNOWNS: usize = 10_000; + + // A shard is `{2-hex}/`, so the space is 0x00..=0xff. + const LAST_SHARD: u16 = 0xff; + + let mut blobs: Vec = Vec::new(); + let mut unknowns: Vec = Vec::new(); + + // Resume in the cursor's own shard — its remaining entries + // still sort after it, and the client-side skip below drops + // the ones that don't. A malformed cursor is a bug in the + // caller's checkpoint, and silently restarting from `00` + // would re-report every blob as new, so refuse it. + let mut shard: u16 = match cursor.as_deref() { + Some(c) => u16::from(u8::from_str_radix(c.get(0..2).unwrap_or(""), 16).map_err( + |_| { + DomainError::internal_error( + "Blob", + format!( + "Azure enumeration cursor '{c}' is not a blob hash — it must \ + start with the two hex characters naming its shard" + ), + ) + }, + )?), + None => 0, + }; + + // Azure caps a page at 5000; asking for the caller's `limit` + // keeps a small page cheap. `MaxResults` rejects zero, and a + // caller asking for nothing still needs a well-formed + // request — and, more importantly, must not be answered with + // an empty page and a `None` cursor, which would read as + // "container fully enumerated, nothing here". + let want = limit.max(1); + let page_size = want.min(5000) as u32; + + 'shards: while shard <= LAST_SHARD { + let prefix = format!("{shard:02x}/"); + // `Pageable` follows `next_marker` itself, so one stream + // covers the whole shard however many round-trips it takes. + let mut pages = self + .container_client + .list_blobs() + .prefix(prefix) + .max_results(std::num::NonZeroU32::new(page_size).expect("clamped above 0")) + .into_stream(); + + while let Some(page) = pages.next().await { + let page = page.map_err(|e| { + DomainError::internal_error( + "Blob", + format!( + "Azure ListBlobs failed on shard {shard:02x} of container '{}': {e}", + self.container_name + ), + ) + })?; + + for blob in page.blobs.blobs() { + let name = blob.name.clone(); + // `OffsetDateTime` → chrono, for the caller's + // grace window. A value outside chrono's range + // degrades to `None`, which the port documents as + // "treat as old enough" — the conservative side, + // since it only ever suppresses a finding on a + // freshly-written blob. + let mtime = chrono::DateTime::::from_timestamp( + blob.properties.last_modified.unix_timestamp(), + blob.properties.last_modified.nanosecond(), + ); + + match Self::hash_from_blob_name(&name) { + Some(hash) => { + // `prefix` is inclusive of the cursor's own + // entry and of everything before it in the + // shard. Without this skip the caller sees + // a hash it already consumed and the + // merge-join never advances past it. + if cursor.as_deref().is_some_and(|c| hash.as_str() <= c) { + continue; + } + blobs.push(BackendBlobEntry { hash, mtime }); + } + // Not ours — a foreign workload sharing the + // container. Surfaced rather than dropped so + // an operator can see it. + None => unknowns.push(BackendUnknownEntry { path: name, mtime }), + } + } + + if blobs.len() >= want { + break 'shards; + } + + if unknowns.len() >= MAX_UNKNOWNS { + return Err(DomainError::internal_error( + "Blob", + format!( + "Azure enumeration accumulated {} non-blob entrie(s) without \ + filling a page, so no resume cursor can be produced. Container \ + '{}' likely holds a large foreign namespace — give OxiCloud a \ + dedicated container.", + unknowns.len(), + self.container_name, + ), + )); + } + } + + shard += 1; + } + + // Exhausting every shard is the ONLY end of enumeration. + // Stopping early because one shard was empty would truncate + // the sweep and report the rest of the container as absent, + // so `shard > LAST_SHARD` — not "this page was empty" — is + // what produces `None`. + let next_cursor = if shard > LAST_SHARD { + None + } else { + blobs.last().map(|entry| entry.hash.clone()) + }; + + Ok(BlobListPage { + blobs, + unknowns, + next_cursor, + }) + }) + } + fn backend_type(&self) -> &'static str { "azure" } @@ -406,14 +652,75 @@ impl BlobStorageBackend for AzureBlobBackend { fn local_blob_path(&self, _hash: &str) -> Option { None } - - // TODO: implement `list_blob_hashes` via - // `container_client.list_blobs()` (`azure_storage_blobs` - // paginator). Same filter as local + S3 impls: - // `/<64-hex>.blob` naming. Currently inherits the trait - // default which returns `operation_not_supported` — the - // `backend_consistency` tenant handles that by emitting a - // single run-level `backend_unenumerable` finding and - // completing without per-blob probes. Ship as a follow-up once - // there's an Azure test environment to validate against. +} + +#[cfg(test)] +mod tests { + use super::*; + + const H: &str = "0a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9"; + + /// The enumeration cursor is fed straight back in as a shard prefix, + /// so a name that does not round-trip would resume in the wrong shard + /// and silently skip everything between. + #[test] + fn blob_name_round_trips_through_hash_from_blob_name() { + let name = AzureBlobBackend::blob_name(H); + assert_eq!(name, format!("0a/{H}.blob")); + assert_eq!( + AzureBlobBackend::hash_from_blob_name(&name).as_deref(), + Some(H) + ); + } + + /// Each of these would otherwise be treated as a hash — and the + /// resume path slices `[0..2]` off it to pick the next shard. + #[test] + fn non_canonical_names_are_rejected() { + let cases = [ + "0a/junk.tmp".to_string(), // spool file + "junk.tmp".to_string(), // no shard + "0a/junk".to_string(), // no suffix + "thumbnails/abc.jpg".to_string(), // foreign namespace + format!("0a/{H}.blob.corrupt"), // sidecar + format!("0a/{H}"), // suffix missing + format!("zz/{H}.blob"), // non-hex shard + format!("ff/{H}.blob"), // shard != hash prefix + format!("0a/{}.blob", &H[..63]), // wrong length + ]; + for name in &cases { + assert_eq!( + AzureBlobBackend::hash_from_blob_name(name), + None, + "must not be read as a blob: {name}" + ); + } + } + + /// The shard walk relies on `{hash[0..2]}/…` ordering lexicographic + /// names into exactly the order `ORDER BY hash` produces. If the + /// shard were not the hash's own prefix the two sequences would + /// interleave differently and the merge-join would emit phantom + /// findings in BOTH directions. + #[test] + fn shard_order_matches_hash_order() { + let hashes = ["00aa", "0a1b", "0aff", "b0cd", "ffff"] + .map(|p| format!("{p}{}", "0".repeat(60))) + .to_vec(); + + let mut names: Vec = hashes + .iter() + .map(|h| AzureBlobBackend::blob_name(h)) + .collect(); + names.sort(); + + let recovered: Vec = names + .iter() + .filter_map(|n| AzureBlobBackend::hash_from_blob_name(n)) + .collect(); + + let mut sorted_hashes = hashes.clone(); + sorted_hashes.sort(); + assert_eq!(recovered, sorted_hashes); + } } diff --git a/src/infrastructure/services/backend_consistency_service.rs b/src/infrastructure/services/backend_consistency_service.rs index ed3a9f70..4ba04798 100644 --- a/src/infrastructure/services/backend_consistency_service.rs +++ b/src/infrastructure/services/backend_consistency_service.rs @@ -1,28 +1,86 @@ //! Fifth tenant of Part 2 (recoverable-run engine). //! -//! Iterates the storage backend's blob-enumeration surface and -//! reports every blob physically present on the backend that has NO -//! matching row in `storage.blobs`. Complements -//! `blobs_consistency` (which walks the DB and probes the backend): -//! together they close the reference graph. +//! **Merge-joins** the backend's blob enumeration against +//! `storage.blobs`, both ordered by hash, so a single pass yields the +//! delta in *both* directions rather than one. //! -//! ### Per-row check +//! It previously walked the backend and probed the DB with +//! `WHERE hash = ANY($1)` over each page, which could only ever see +//! backend-only entries: a row whose bytes are gone never appears in a +//! backend listing, so it was invisible here by construction. That half +//! was left to `blobs_consistency`'s per-row HEAD probe, which does not +//! survive the row counts this plan produces — see +//! `docs/plan/derived-blobs.md`. That probe is now gone: this tenant +//! owns every backend-side check, and `blobs_consistency` is DB-only. +//! +//! ### Per-row checks //! //! * `orphan_blob` (severity `inconsistent`) — bytes on disk / S3 / //! Azure with no registry row. Not data-loss (nothing broken — //! just storage overhead), but points at dedup_gc or //! ingest-path drift. Recovery = register-registry-row (if the //! bytes are still needed) OR delete the file (if truly orphan). +//! * `blob_missing_from_backend` (severity `data_loss`) — a registry +//! row whose bytes are absent. The opposite direction and the more +//! serious one: an orphan wastes space, this loses a file. +//! * `blob_corrupted` (severity `data_loss`, `?deep=true` only) — +//! the key exists on both sides but the bytes behind it no longer +//! hash to it. Silent bit-rot. +//! * `blob_unreadable` (severity `data_loss`, `?deep=true` only) — +//! the key exists but the bytes cannot be read at all: decrypt +//! failure (missing key), transport error, permissions. Same impact +//! as corruption from a file's point of view, different remedy, +//! hence a separate kind. Triage on the recorded `error`. //! -//! ### Run-level check +//! ### Deep mode //! -//! * `backend_unenumerable` (severity `anomaly`) — the backend -//! returned `operation_not_supported` on the first -//! `list_blob_hashes` call. Currently this fires when a -//! `MigrationBlobBackend` is active (refuses enumeration -//! mid-migration by design) or on an Azure backend (Azure impl -//! deferred). Informational — operators know they can't rely on -//! this scan under that config. +//! The last two moved here from `blobs_consistency`, which used to +//! carry a backend solely for them. Re-hashing is backend work end to +//! end — the only DB input is the hash — and this walk already holds +//! the matched key pairs, which is exactly the set worth reading. It +//! costs a full read of every blob, so it is opt-in. +//! +//! ### Why the two orderings agree +//! +//! The merge-join's premise is that the backend's byte order and the +//! database's `ORDER BY hash` rank identically. They do, because hashes +//! are lowercase BLAKE3 hex of fixed length: over `[0-9a-f]` digits +//! precede letters in both, and there is no case to fold. A hash column +//! that ever admitted uppercase or variable length would break this +//! silently and in both directions at once. +//! +//! ### When enumeration fails +//! +//! **The run fails.** There is no degraded mode. +//! +//! There used to be: an error on the first `list_blob_hashes` call +//! emitted a `backend_unenumerable` anomaly and fell back to +//! `probe_each_row`, one `blob_exists` per `storage.blobs` row. That +//! recovered `blob_missing_from_backend` (the direction that loses +//! FILES) but never `orphan_blob`, since bytes no row claims are +//! invisible to anything starting from the database. +//! +//! It was written for two cases, and neither exists: +//! +//! * **Azure** — enumerates since the 256-way shard walk (see +//! `AzureBlobBackend::list_blob_hashes` for why it needs one to do +//! what S3 gets from `StartAfter`). +//! * **Mid-migration** — never applied. That justification named a +//! `MigrationBlobBackend` that does not exist; +//! `SwappableBlobBackend::list_blob_hashes` forwards to whatever is +//! currently active, as do the Encrypted, Cached and Retry wrappers. +//! Do not reintroduce the claim without grepping for the impl. +//! +//! So the only thing still reaching it was a *transient* failure — auth +//! blip, throttle, network — being relabelled as a capability limit and +//! silently costing orphan coverage. A failed run is louder than an +//! anomaly on an otherwise-clean-looking scan, which was the fallback's +//! own stated goal. +//! +//! The trait default still returns `operation_not_supported`, so a +//! future write-only or read-only-mirror backend would fail every run +//! here. **That is when the fallback should come back — with tests.** +//! It had none, which is the other half of why it went. //! //! ### Grace window //! @@ -42,7 +100,6 @@ //! denominator (backend count ≈ blob count on a healthy install; //! deviation IS the finding). -use std::collections::HashSet; use std::sync::Arc; use async_trait::async_trait; @@ -54,14 +111,18 @@ use crate::infrastructure::scheduler::{ JobRegistry, JobRunArgs, JobStore, JobStoreProvider, ProgressKind, RecoverableJobHandler, RunOutcome, RunStatus, record_or_log, }; +use crate::infrastructure::services::blob_diagnostics::affected_files; pub const BACKEND_CONSISTENCY_JOB_NAME: &str = "backend_consistency"; -/// Same `params` JSONB key `blobs_consistency` uses — kept identical -/// so operators grepping run rows see the same convention across -/// both storage-audit tenants. -pub const PROBED_STORAGE_PARAM: &str = - crate::infrastructure::services::blobs_consistency_service::PROBED_STORAGE_PARAM; +/// `params` JSONB key under which the entry name being enumerated is +/// stashed on a Fresh run (matches `TARGET_NAME_PARAM` on +/// `backend_migration`). Resumed runs re-read it so a paused audit +/// survives restart without the admin re-specifying the target. +/// +/// Defined here rather than in `blobs_consistency`, which no longer +/// touches a backend and so has no entry to scope. +pub const PROBED_STORAGE_PARAM: &str = "probed_storage"; /// Batch size for backend enumeration + DB probe. 500 is enough to /// amortise the DB round-trip while keeping the cancel-poll cadence @@ -78,10 +139,6 @@ const BATCH_SIZE: usize = 500; /// `blobs_consistency` + `dedup_gc`. const CREATE_GRACE: Duration = Duration::hours(1); -/// Cap on affected-blob examples surfaced in the run-level -/// `backend_unenumerable` finding. Keeps the finding detail bounded. -const _MAX_EXAMPLES: usize = 5; - pub struct BackendConsistencyCheck { pool: Arc, /// Default backend to enumerate when `args.storage` is `None` — @@ -130,6 +187,18 @@ impl RecoverableJobHandler for BackendConsistencyCheck { BACKEND_CONSISTENCY_JOB_NAME } + fn description(&self) -> &'static str { + "Merge-joins the storage backend's blob enumeration against \ + storage.blobs, both ordered by hash, so one pass yields the delta \ + in both directions: bytes on the backend no DB row claims, and \ + rows whose bytes are gone. If the backend cannot be enumerated \ + the run fails rather than reporting partial coverage. \ + Add ?deep=true to also read every matched blob back and re-hash \ + it, catching silent bit-rot — that is a full read of storage and \ + can take hours. Read-only in every mode: nothing is uploaded or \ + deleted." + } + /// Approximate total: on a healthy install every backend blob /// has a `storage.blobs` row, so the DB count is a proxy for /// the backend count. The fraction deviating from 1.0 at run @@ -234,6 +303,49 @@ impl RecoverableJobHandler for BackendConsistencyCheck { ); } + // Deep mode — read every matched blob back and re-hash it, rather + // than trusting that a key present on both sides means the bytes + // behind it are still the bytes that key names. + // + // It lives here rather than in `blobs_consistency` because it is + // a backend operation end to end: the only DB input is the hash, + // which this merge-join already holds. Keeping it there forced + // that tenant to carry a backend for one flag, which is the + // overlap this split removes. + // + // Persisted to `params.deep` on a Fresh run so a Resume picks up + // the same mode (a Paused deep scan must not silently continue + // shallow) and the admin run-detail view can show what the scan + // actually verified. Written BEFORE the walk so a crash mid-batch + // still leaves the marker. + let deep = if is_fresh { + let v = if args.deep { "true" } else { "false" }; + if let Err(e) = store.set_string_param("deep", v).await { + return RunOutcome::Failed { + message: format!("failed to persist deep flag to params: {e}"), + }; + } + args.deep + } else { + match store.get_string_param("deep").await { + Ok(Some(v)) => v == "true", + Ok(None) => false, + Err(e) => { + return RunOutcome::Failed { + message: format!("read `deep` from params: {e}"), + }; + } + } + }; + if deep { + tracing::info!( + target: "oxicloud::consistency", + event = "backend_consistency.deep_mode_active", + run_id = %store.run_id(), + "deep mode: re-reading + re-hashing every matched blob (bit-rot detection)" + ); + } + // Cursor = opaque backend continuation token, UTF-8-encoded. // Each backend defines its own format (local = shard/hash, // S3 = ListObjectsV2 continuation token, Azure = list @@ -286,45 +398,25 @@ impl RecoverableJobHandler for BackendConsistencyCheck { let page = match backend.list_blob_hashes(cursor.clone(), BATCH_SIZE).await { Ok(v) => v, Err(e) => { - // Backend refuses / can't enumerate. First-batch - // failure = we emit ONE run-level anomaly and - // complete cleanly (the run stays useful — the - // operator learns why nothing was checked - // instead of getting a red error). Mid-scan - // failure = we fail the run. - - let is_first_batch = cursor.is_none() && finding_count == 0; - if is_first_batch { - // No local increment — the local - // `finding_count` is only used for the - // completion log below, but this branch - // returns immediately. The finding IS - // persisted + counted in `stats.finding_count` - // by `record_or_log` → `store.record_finding`. - record_or_log( - store, - BACKEND_CONSISTENCY_JOB_NAME, - "backend_unenumerable", - "anomaly", - None, - serde_json::json!({ - "backend": backend.backend_type(), - "error": format!("{e}"), - "note": "backend refused enumeration; no per-blob orphan probes attempted", - }), - ) - .await; - tracing::info!( - target: "oxicloud::consistency", - event = "backend_consistency.unenumerable", - run_id = %store.run_id(), - backend = backend.backend_type(), - "backend refused enumeration (typical during migration or on backends without list support)" - ); - return RunOutcome::completed(); - } + // Fail loudly, first batch or not. + // + // A first-batch failure used to degrade to + // `probe_each_row` instead. That was written for + // backends which genuinely cannot enumerate, and none + // ship today — see the module docs for why the two it + // named do not apply. What was left reaching it was a + // transient error relabelled as a capability limit, on + // a run that then looked clean while having lost orphan + // coverage entirely. + // + // Whether the enumeration died on page 1 or page 900, + // the audit did not complete, and the operator needs to + // know that rather than read a green run. return RunOutcome::Failed { - message: format!("backend list failed mid-scan: {e}"), + message: format!( + "backend enumeration failed on {}: {e}", + backend.backend_type() + ), }; } }; @@ -376,60 +468,174 @@ impl RecoverableJobHandler for BackendConsistencyCheck { return RunOutcome::completed(); } - // Batch DB probe: which of these hashes have a - // `storage.blobs` row? One `WHERE hash = ANY($1)` per - // batch — indexed lookup, cheap even on millions of - // rows. - let batch_hashes: Vec = page.blobs.iter().map(|e| e.hash.clone()).collect(); - let db_present: HashSet = if batch_hashes.is_empty() { - HashSet::new() - } else { - match sqlx::query_as::<_, (String,)>( - r#"SELECT hash FROM storage.blobs WHERE hash = ANY($1)"#, - ) - .bind(&batch_hashes[..]) - .fetch_all(self.pool.as_ref()) - .await - { - Ok(rows) => rows.into_iter().map(|(h,)| h).collect(), - Err(e) => { - return RunOutcome::Failed { - message: format!("db probe: {e}"), - }; - } + // ── Merge-join, not a one-sided probe ──────────────── + // + // Both sides are ordered by hash ascending — the backend by + // contract (`BlobStorageBackend::list_blob_hashes`), the DB by + // `ORDER BY hash` — so one pass yields BOTH deltas instead of + // one: + // + // * present on the backend, absent from the DB → `orphan_blob` + // * present in the DB, absent from the backend → + // `blob_missing_from_backend` (data loss, not overhead) + // + // The old form probed `WHERE hash = ANY($1)` over the backend + // page, so it could only ever see the first kind: a row whose + // bytes are gone never appears in a backend listing and was + // invisible here by construction. + // + // Ordering is the whole premise, so it is worth being explicit + // about why the two agree. Hashes are lowercase BLAKE3 hex of + // fixed length, and over `[0-9a-f]` the database collation and + // byte order rank identically (digits before letters in both, + // no case folding to disagree about). A hash column that ever + // admitted uppercase or variable length would break this + // silently, in both directions. + let db_hashes: Vec = match sqlx::query_as::<_, (String,)>( + r#"SELECT hash FROM storage.blobs + WHERE ($1::text IS NULL OR hash > $1) + ORDER BY hash + LIMIT $2"#, + ) + .bind(cursor.as_deref()) + .bind(BATCH_SIZE as i64) + .fetch_all(self.pool.as_ref()) + .await + { + Ok(rows) => rows.into_iter().map(|(h,)| h).collect(), + Err(e) => { + return RunOutcome::Failed { + message: format!("db page: {e}"), + }; } }; - for entry in &page.blobs { - if db_present.contains(&entry.hash) { - continue; - } - if let Some(mtime) = entry.mtime - && mtime > grace_cutoff - { - continue; - } + // The two pages cover different ranges, so only the overlap can + // be judged. Beyond `horizon` a hash missing from one side may + // simply be on the next page of the other, and emitting there + // would invent findings in both directions. When a side is + // exhausted its entries cannot be "on a later page", so the + // other side's tail becomes judgeable. + let backend_last = page.blobs.last().map(|e| e.hash.as_str()); + let db_last = db_hashes.last().map(|s| s.as_str()); + let backend_done = page.next_cursor.is_none(); + let db_done = db_hashes.len() < BATCH_SIZE; - finding_count += 1; - record_or_log( - store, - BACKEND_CONSISTENCY_JOB_NAME, - "orphan_blob", - "inconsistent", - None, - serde_json::json!({ - "hash": entry.hash, - "mtime": entry.mtime.map(|t| t.to_rfc3339()), - "backend": backend.backend_type(), - }), - ) - .await; + let horizon: Option<&str> = match (backend_last, db_last) { + _ if backend_done && db_done => None, // judge everything + (Some(b), Some(d)) if backend_done => Some(b.max(d)), + (Some(b), Some(d)) if db_done => Some(b.max(d)), + (Some(b), Some(d)) => Some(b.min(d)), + (Some(b), None) => Some(b), + (None, Some(d)) => Some(d), + (None, None) => None, + }; + let in_range = |h: &str| horizon.is_none_or(|limit| h <= limit); + + let mut bi = page.blobs.iter().peekable(); + let mut di = db_hashes.iter().peekable(); + loop { + match (bi.peek(), di.peek()) { + // Present on both sides. Shallow: nothing to say — the + // key exists where the registry claims. Deep: the key + // matching says nothing about the bytes behind it, so + // read them back and re-hash. + // + // Guarded by `in_range` so a pair past the horizon is + // not read twice — the cursor stops at the horizon, so + // that pair comes round again next batch and is + // verified then. + (Some(b), Some(d)) if b.hash == **d => { + if deep && in_range(&b.hash) { + finding_count += + self.verify_bytes(store, backend.as_ref(), &b.hash).await; + } + bi.next(); + di.next(); + } + // Backend-only: bytes with no registry row. + (Some(b), d_opt) + if d_opt.is_none_or(|d| b.hash.as_str() < d.as_str()) + && in_range(&b.hash) => + { + // Grace window: the write path is + // durability-before-visibility, so bytes exist + // briefly before their row does. Without this every + // in-flight upload reads as an orphan. + if !matches!(b.mtime, Some(m) if m > grace_cutoff) { + finding_count += 1; + record_or_log( + store, + BACKEND_CONSISTENCY_JOB_NAME, + "orphan_blob", + "inconsistent", + None, + serde_json::json!({ + "hash": b.hash, + "mtime": b.mtime.map(|t| t.to_rfc3339()), + "backend": backend.backend_type(), + }), + ) + .await; + } + bi.next(); + } + // DB-only: a row whose bytes are gone. Severity is + // `data_loss`, not `inconsistent` — an orphan wastes + // space, this loses a file. + (b_opt, Some(d)) + if b_opt.is_none_or(|b| d.as_str() < b.hash.as_str()) && in_range(d) => + { + finding_count += 1; + record_or_log( + store, + BACKEND_CONSISTENCY_JOB_NAME, + "blob_missing_from_backend", + "data_loss", + None, + serde_json::json!({ + "hash": d, + "backend": backend.backend_type(), + "note": "registry row with no bytes on the backend", + }), + ) + .await; + di.next(); + } + // Past the horizon on both sides, or both exhausted. + _ => break, + } } - // Advance cursor + checkpoint. Scanned count tracks - // both blobs and unknowns since we walked both. + // Advance to the horizon, not the backend's own cursor. + // + // One hash serves both sides: they share an ordering, so "resume + // after H" means `start_after(H)` on the backend and + // `WHERE hash > H` in the DB. Advancing past the horizon would + // skip the un-judged tail of whichever side reached further. + // + // Scanned count covers blobs and unknowns, since both were + // walked. let batch_len = (page.blobs.len() + page.unknowns.len()) as u64; - cursor = page.next_cursor; + let exhausted = backend_done && db_done; + cursor = if exhausted { + None + } else { + horizon.map(|h| h.to_string()) + }; + + // Neither side exhausted yet no horizon means neither returned a + // row — nothing left to compare, and continuing would spin on the + // same empty pages forever. + if cursor.is_none() && !exhausted && horizon.is_none() { + tracing::debug!( + target: "oxicloud::consistency", + event = "backend_consistency.no_horizon", + run_id = %store.run_id(), + "both sides returned no rows before exhaustion; ending the sweep" + ); + } + let cursor_bytes = cursor .as_ref() .map(|s| s.as_bytes().to_vec()) @@ -440,8 +646,7 @@ impl RecoverableJobHandler for BackendConsistencyCheck { }; } - // Backend returned no next_cursor → enumeration - // complete. Emit the completion log and return. + // Both sides drained → the sweep is complete. if cursor.is_none() { tracing::info!( target: "oxicloud::consistency", @@ -456,3 +661,106 @@ impl RecoverableJobHandler for BackendConsistencyCheck { } } } + +impl BackendConsistencyCheck { + /// Deep-mode per-blob verification. Reads the blob back, re-hashes it, + /// and records what it finds. Returns the number of findings recorded + /// (0 or 1) so the caller's counter stays the single tally. + /// + /// Moved here from `blobs_consistency` along with the rest of the + /// backend-touching work: the merge-join already holds a verified + /// key pair, which is exactly the set worth reading. + async fn verify_bytes( + &self, + store: &dyn JobStore, + backend: &dyn BlobStorageBackend, + hash: &str, + ) -> u64 { + match recompute_hash(backend, hash).await { + // The bytes still hash to the key they are filed under. + Ok(computed) if computed == hash => 0, + // Silent bit-rot. `computed_hash` is reported rather than a + // bare "mismatch" because the value is diagnostic: a one-bit + // flip, a truncation and a whole-object swap leave distinct + // signatures. + Ok(computed) => { + let affected = affected_files(self.pool.as_ref(), hash).await; + record_or_log( + store, + BACKEND_CONSISTENCY_JOB_NAME, + "blob_corrupted", + "data_loss", + None, + serde_json::json!({ + "hash": hash, + "computed_hash": computed, + "backend": backend.backend_type(), + "affected_files": affected, + }), + ) + .await; + 1 + } + // Bytes are there by key but cannot be read at all: decrypt + // failure (missing key), transport error, permissions. Same + // impact as corruption from a file's point of view — the + // content is inaccessible — but a different remedy, which is + // why it is a separate kind rather than folded into + // `blob_corrupted`. Operators triage on `error`. + Err(e) => { + let affected = affected_files(self.pool.as_ref(), hash).await; + record_or_log( + store, + BACKEND_CONSISTENCY_JOB_NAME, + "blob_unreadable", + "data_loss", + None, + serde_json::json!({ + "hash": hash, + "backend": backend.backend_type(), + "affected_files": affected, + "error": e.to_string(), + }), + ) + .await; + tracing::warn!( + target: "oxicloud::consistency", + event = "backend_consistency.blob_unreadable", + run_id = %store.run_id(), + hash = %hash, + error = %e, + "🚨 blob unreadable in deep mode — recorded finding, continuing" + ); + 1 + } + } + } +} + +/// Deep-mode helper — read the blob from the backend and recompute its +/// BLAKE3 hash. Returns the recomputed hex string; callers compare it +/// against the expected hash themselves. Returning the actual hash (not +/// a bool) lets the finding surface WHAT the bytes now hash to, which is +/// diagnostic gold: a one-bit flip has a very different signature from a +/// chunk-boundary corruption or a truncated read. `Err(_)` on any +/// backend-side error — the caller records that as `blob_unreadable` +/// rather than as corruption. +async fn recompute_hash( + backend: &dyn BlobStorageBackend, + expected_hash: &str, +) -> Result { + use crate::common::errors::DomainError; + use futures::StreamExt; + + let mut stream = backend.get_blob_stream(expected_hash).await?; + let mut hasher = blake3::Hasher::new(); + + while let Some(chunk) = stream.next().await { + let bytes = chunk.map_err(|e| { + DomainError::internal_error("BackendConsistency", format!("stream read: {e}")) + })?; + hasher.update(&bytes); + } + + Ok(hasher.finalize().to_hex().to_string()) +} diff --git a/src/infrastructure/services/backend_migration_service.rs b/src/infrastructure/services/backend_migration_service.rs index 61dabc72..9eb56347 100644 --- a/src/infrastructure/services/backend_migration_service.rs +++ b/src/infrastructure/services/backend_migration_service.rs @@ -59,8 +59,8 @@ use crate::application::ports::blob_storage_ports::BlobStorageBackend; use crate::common::config::NamedStorageEntry; use crate::common::errors::DomainError; use crate::infrastructure::scheduler::{ - JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome, - RunStatus, record_or_log, + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, Mutates, RecoverableJobHandler, + RunOutcome, RunStatus, record_or_log, }; use crate::infrastructure::services::encrypted_blob_backend::{EncryptedBlobBackend, HeadCheck}; use crate::infrastructure::services::entry_backend::{ @@ -188,6 +188,20 @@ impl RecoverableJobHandler for BackendMigrationService { BACKEND_MIGRATION_JOB_NAME } + fn description(&self) -> &'static str { + "Copies every blob payload from the backend the server booted with \ + to the one the current storage settings describe. Covers legacy \ + whole-file blobs and CDC chunks in a single walk. Resumable — a \ + paused or crashed run continues from its cursor rather than \ + restarting." + } + + /// Writes bytes to the target backend. Source bytes are left in place — + /// the copy is additive, so an aborted migration loses nothing. + fn mutates(&self) -> Mutates { + Mutates::Always + } + /// Definitive count — one row per blob. `SELECT COUNT(*) FROM /// storage.blobs` on a modern PG is a sub-second index-only scan /// even at millions of rows. @@ -894,14 +908,14 @@ impl BackendMigrationService { source_missing = source_missing, "🛑 backend_migration aborted — {failed} blob(s) failed, active backend left at \ `{previous_active}`, readonly cleared. Inspect findings and retry, or accept \ - the partial migration via `oxicloud --select-storage {target_name}`." + the partial migration via `oxicloud storage select {target_name}`." ); return RunOutcome::Failed { message: format!( "{failed} blob(s) failed to migrate — active backend NOT switched \ (still `{previous_active}`). Retry the run (short-circuits on already-copied \ blobs) or accept the partial migration manually via \ - `oxicloud --select-storage {target_name}`." + `oxicloud storage select {target_name}`." ), }; } diff --git a/src/infrastructure/services/backend_rotate_service.rs b/src/infrastructure/services/backend_rotate_service.rs index e428cce2..074a35bf 100644 --- a/src/infrastructure/services/backend_rotate_service.rs +++ b/src/infrastructure/services/backend_rotate_service.rs @@ -62,8 +62,8 @@ use crate::application::ports::blob_storage_ports::BlobStorageBackend; use crate::common::config::NamedStorageEntry; use crate::common::migration_progress::MigrationProgress; use crate::infrastructure::scheduler::{ - JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome, - RunStatus, record_or_log, + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, Mutates, RecoverableJobHandler, + RunOutcome, RunStatus, record_or_log, }; use crate::infrastructure::services::encrypted_blob_backend::BlobFormat; use crate::infrastructure::services::entry_backend::build_entry_backend_typed; @@ -135,6 +135,20 @@ impl RecoverableJobHandler for BackendRotateService { BACKEND_ROTATE_JOB_NAME } + fn description(&self) -> &'static str { + "Brings every blob's on-disk format in line with the storage \ + entry's current head key: encrypts plaintext, re-encrypts under a \ + rotated key, decrypts when the head is 'none', and upgrades \ + legacy blobs to v1. Blobs already in the right format are skipped, \ + so re-running after a key change is cheap." + } + + /// Rewrites blobs **in place**. Unlike a migration this has no additive + /// fallback — the previous ciphertext is gone once a blob is rewritten. + fn mutates(&self) -> Mutates { + Mutates::Always + } + /// Definitive count — one row per blob. Same query as /// `backend_migration::count_total`; the two walk the same rows. async fn count_total(&self) -> Option { diff --git a/src/infrastructure/services/blob_diagnostics.rs b/src/infrastructure/services/blob_diagnostics.rs new file mode 100644 index 00000000..a5639910 --- /dev/null +++ b/src/infrastructure/services/blob_diagnostics.rs @@ -0,0 +1,46 @@ +//! Reverse-lookup helpers shared by the storage consistency tenants. +//! +//! `blobs_consistency` (DB-side: refcount drift) and +//! `backend_consistency` (backend-side: missing / orphaned / corrupted +//! bytes) both answer the same operator question when they emit a +//! finding — *which files does this hash break?* — so the query lives +//! here rather than in whichever tenant happened to need it first. + +use sqlx::PgPool; + +/// Cap on reverse-lookup file names surfaced in a finding's detail. +/// Keeps detail JSON bounded when a broken blob is referenced by +/// hundreds of files. +const AFFECTED_FILES_SAMPLE: i64 = 5; + +/// Sample of file names that reference this blob — either directly +/// (`files.blob_hash = $hash`, legacy pre-CDC) or transitively via a +/// manifest (`chunk_hashes @> ARRAY[$hash]`, the post-CDC dominant +/// path). Capped so a chunk shared by 10 000 files doesn't blow up the +/// finding detail JSON. Order is arbitrary — this samples for +/// diagnosis, it does not enumerate. +/// +/// Returns an empty vec on query error: a finding with no sample is +/// still a finding, and failing the sweep because the diagnostic +/// garnish didn't load would trade the whole scan for a nicety. +pub(crate) async fn affected_files(pool: &PgPool, hash: &str) -> Vec { + let rows: Vec<(String,)> = sqlx::query_as( + r#" + SELECT DISTINCT f.name + FROM storage.files f + WHERE f.blob_hash = $1 + OR EXISTS ( + SELECT 1 FROM storage.chunk_manifests m + WHERE m.file_hash = f.blob_hash + AND $1 = ANY(m.chunk_hashes) + ) + LIMIT $2 + "#, + ) + .bind(hash) + .bind(AFFECTED_FILES_SAMPLE) + .fetch_all(pool) + .await + .unwrap_or_default(); + rows.into_iter().map(|(n,)| n).collect() +} diff --git a/src/infrastructure/services/blobs_consistency_service.rs b/src/infrastructure/services/blobs_consistency_service.rs index 5f0c3a0e..6fdd248c 100644 --- a/src/infrastructure/services/blobs_consistency_service.rs +++ b/src/infrastructure/services/blobs_consistency_service.rs @@ -1,34 +1,15 @@ //! Fourth tenant of Part 2 (recoverable-run engine). //! -//! Iterates `storage.blobs` — the content-addressable registry — -//! and verifies each row against the physical backend AND against -//! the reference-counting invariants that `dedup_gc` relies on. +//! Iterates `storage.blobs` — the content-addressable registry — and +//! checks the reference-counting invariants `dedup_gc` relies on. //! -//! Three per-row checks (subject-iteration principle in action — -//! one walk, multiple branches): +//! **Database only.** It opens no backend and makes no network call; +//! `?storage=` and `?deep=true` are both inert here. //! -//! * `blob_missing_from_backend` (severity `data_loss`) — the DB -//! row says the hash exists but `BlobStorageBackend::blob_exists` -//! returns false. Bytes gone from disk / S3 / Azure. Any file -//! whose manifest references this hash (or whose whole-file -//! `blob_hash` points at it) will fail to read. +//! Two per-row checks share the same walk — one page fetch already +//! has every column both need: //! -//! * `blob_corrupted` (severity `data_loss`, deep mode only) — -//! bytes exist on the backend but their BLAKE3 no longer matches -//! the hash under which they're indexed. Silent bit-rot. Only -//! runs when the operator passes `?deep=true` because it costs a -//! full read of every blob. -//! -//! * `blob_unreadable` (severity `data_loss`, deep mode only) — -//! `blob_exists` returned true but the read pipeline errored (can't -//! decrypt, network glitch, permission error, etc.). Distinct from -//! `blob_corrupted` (which requires successful read + hash mismatch); -//! here we can't get bytes out at all. Same operator impact — any -//! file referencing this hash is inaccessible — but the remedy -//! differs (key recovery, retry, or blob replacement, depending on -//! the recorded `error` field). -//! -//! * `refcount_mismatch` (severity `inconsistent`) — +//! * `refcount_mismatch` (severity `inconsistent`, repairable) — //! `storage.blobs.ref_count` disagrees with the actual reference //! count computed from `storage.files.blob_hash` + //! `storage.chunk_manifests.chunk_hashes[]`. Under-count means @@ -36,98 +17,186 @@ //! a blob is being pinned longer than needed. Content-safe either //! way (the storage.blobs row is fine, the counter is wrong). //! -//! ### Complements `files_consistency` +//! * `blob_orphan_stalled` (severity `anomaly`, discovery-only) — +//! the row satisfies every reap predicate `dedup_gc` uses +//! (`ref_count <= 0`, no chunk-level referrer) AND has been sitting +//! past a comfortable margin (default `STALL_GRACE_SECS` = 24 h, +//! comfortably exceeding the GC's own 1 h grace). Signal that the +//! GC pipeline itself is stuck — the job stopped running, is +//! failing on the same hash every tick, or a ghost row keeps +//! pinning the same set. No `?repair=true` path: per +//! [[feedback_no_silent_auto_repair]], papering over the symptom +//! here would hide the root cause (a wedged worker, a hanging +//! backend delete, a ghost referrer being recreated) — the +//! operator diagnoses first, then runs +//! `POST /api/admin/jobs/dedup_gc/trigger?force=true` themselves. +//! The two checks are orthogonal in-loop: a row with drift is NOT +//! also flagged as stalled — the drift IS why the GC hasn't taken +//! it, so fixing the counter is the whole story. //! -//! `files_consistency` (Slice 6/10) iterates files and verifies DB -//! integrity. `blobs_consistency` iterates the storage registry and -//! verifies physical existence + counter integrity. Together they -//! cover both sides of the reference graph. Neither doubles the -//! other's work — probing per-blob (here) instead of per-file-chunk -//! preserves dedup savings: a chunk shared by 5 files gets probed -//! ONCE. +//! ### Why nothing physical lives here any more //! -//! ### Not covered here +//! This tenant used to probe `BlobStorageBackend::blob_exists` once +//! per row for `blob_missing_from_backend`, and under `?deep=true` +//! read and re-hashed every blob for `blob_corrupted` / +//! `blob_unreadable`. //! -//! * **Orphan bytes on the backend** (files on disk with no DB row) -//! — belongs in the future `backend_consistency` tenant which -//! iterates the backend itself. Requires the `list_blob_hashes` -//! trait extension and per-backend enumeration impls. +//! All three moved to `backend_consistency`, which merge-joins the +//! backend's enumeration against this same table in one ordered pass. +//! It reports the same missing bytes, plus the backend-only orphans a +//! DB walk cannot see by construction, at one enumeration instead of +//! N round-trips — and a deep pass there re-hashes the matched pairs +//! it already holds. Keeping the probe here bought nothing and made +//! every scheduled sweep pay for it. +//! +//! What is left is the half that needs no backend at all: a counter, +//! and the two tables that determine what it should be. +//! +//! ### Elsewhere in the graph +//! +//! * **Physical existence, orphan bytes, bit-rot** — +//! `backend_consistency`. +//! * **File-side DB integrity** (parent folder, blob reference, +//! denormalised size) — `files_consistency`. //! * **Manifest-level integrity** (`storage.chunk_manifests` rows -//! pointing at reaped chunks) — already covered by -//! `files_consistency::chunk_missing`. +//! pointing at reaped chunks) — `files_consistency::chunk_missing`. +//! * **The OTHER refcount** (`chunk_manifests.ref_count`, which every +//! whole-Blob reference lands on) — `manifests_consistency`. -use std::path::PathBuf; use std::sync::Arc; use async_trait::async_trait; -use chrono::{DateTime, Duration, Utc}; +use chrono::{DateTime, Utc}; use sqlx::PgPool; -use crate::application::ports::blob_storage_ports::BlobStorageBackend; -use crate::common::config::NamedStorageEntry; +use crate::application::ports::blob_reference_ports::{BlobReferenceRegistry, RefLevel}; use crate::infrastructure::scheduler::{ - JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome, - RunStatus, record_or_log, + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, Mutates, RecoverableJobHandler, + RunOutcome, RunStatus, record_or_log, }; -use crate::infrastructure::services::entry_backend::build_entry_backend; +use crate::infrastructure::services::blob_diagnostics::affected_files; pub const BLOBS_CONSISTENCY_JOB_NAME: &str = "blobs_consistency"; -/// `params` JSONB key under which the entry name being probed is -/// stashed on a Fresh run (matches `TARGET_NAME_PARAM` on -/// `backend_migration`). Resumed runs re-read it so a paused audit -/// survives restart without the admin re-specifying the target. -pub const PROBED_STORAGE_PARAM: &str = "probed_storage"; - /// Rows per batch. Blobs are numerous (millions on a busy install) -/// but per-row work is one indexed backend probe + one indexed SQL -/// ref-count query. 200 balances cancel-poll cadence against -/// round-trip amortisation. +/// but per-row work is now a single indexed SQL ref-count comparison, +/// with no backend round-trip. 200 balances cancel-poll cadence +/// against round-trip amortisation. const BATCH_SIZE: i64 = 200; -/// Grace window — rows created within this window are skipped by -/// the physical-existence probe because the write path is -/// durability-before-visibility: `dedup_service` writes bytes, then -/// registers the row a few ms later. A scan catching a row -/// mid-write would false-positive it as `blob_missing_from_backend`. -/// Same shape `dedup_gc` uses (see its `grace_secs`). -const CREATE_GRACE: Duration = Duration::hours(1); - -/// Cap on reverse-lookup file names surfaced in a finding's detail. -/// Keeps detail JSON size bounded when a broken blob is referenced -/// by hundreds of files. -const AFFECTED_FILES_SAMPLE: i64 = 5; +/// Grace window for the `blob_orphan_stalled` check. Derived from +/// `dedup_gc`'s own grace so the two stay coupled at the source: if an +/// operator ever tunes GC's grace (e.g. long-network-upload profile), +/// the stall threshold auto-scales — no second knob to keep in sync. +/// +/// The `× 24` multiplier says "we tolerate up to 24 missed sweep ticks +/// before crying stall". Discovery-only, so a false positive after a +/// long maintenance pause costs nothing (operator glances at the +/// finding, sees it clear on the next run, moves on). +const STALL_GRACE_SECS: i64 = + crate::infrastructure::services::dedup_service::DedupService::GC_ORPHAN_GRACE_SECS * 24; pub struct BlobsConsistencyCheck { pool: Arc, - /// The default backend to probe when `args.storage` is `None` — - /// the currently-active LIVE backend, injected at DI time. Runs - /// with `?storage=` build a fresh backend for the named - /// entry instead (via [`build_entry_backend`]). - backend: Arc, - /// Snapshot of `AppConfig.storage_entries` used to resolve - /// `args.storage` to a `NamedStorageEntry`. Empty for the - /// legacy zero-entries path — `?storage=` runs then - /// fail-fast with a clear "no entries declared" message. - storage_entries: Vec, - /// Ambient `AppConfig.storage_path` — used as the `root_dir` - /// fallback for a Local target entry with no `_ROOT_DIR`. Same - /// fallback rule the boot path uses. - storage_path_fallback: PathBuf, + /// The chunk-level page query, assembled once from the blob-reference + /// registry so this recompute and `dedup_gc` agree on what "referenced" + /// means. Built at construction rather than per page so the sweep runs a + /// fixed statement — same reasoning as `DedupService::manifest_reap_sql`. + /// See `docs/plan/derived-blobs.md`. + chunk_page_sql: String, + /// Per-row repair UPDATE, built from the SAME registry as + /// `chunk_page_sql` so detection and repair use identical formulas + /// by construction. A future `RefLevel::Chunk` ref source added to + /// the registry flows into both without a code change here. + /// + /// Previously the repair query was inlined with a hardcoded + /// 2-term formula (accidentally matching detection today). Would + /// silently diverge the moment a new chunk-level ref source + /// landed — same class of latent bug the sibling + /// `manifests_consistency` service hit 2026-09-02. Preemptively + /// pulled from the registry here to keep the pair symmetric. + chunk_repair_sql: String, +} + +/// The chunk-level page query, with `actual_ref_count` summed from the +/// registered reference sources. +/// +/// `storage.blobs.ref_count` semantics — the invariant `dedup_service` +/// actually maintains: +/// +/// ```text +/// ref_count = (number of chunk_manifests whose chunk_hashes[] contains +/// this hash) +/// + (number of files.blob_hash pointing at this hash on the +/// LEGACY whole-file path — files with NO manifest for their +/// blob_hash) +/// ``` +/// +/// Naively `COUNT(files) + COUNT(manifests referring)` double-counts +/// single-chunk CDC files: where a file's whole-file hash equals its lone +/// chunk's hash (anything under one CDC chunk), the file appears BOTH in +/// `files.blob_hash` and in the manifest's `chunk_hashes[]`. The +/// `NOT EXISTS` guard inside `FilesReferenceSource`'s chunk-level fragment +/// excludes CDC-path files from the legacy term so the two don't overlap. +/// +/// The GIN index on `chunk_hashes` (migration +/// `20260628000000_delta_upload_gin_index`) keeps the `= ANY(chunk_hashes)` +/// probe cheap. +/// +/// # Panics +/// +/// If no source contributes at [`RefLevel::Chunk`] — a wiring bug that +/// would make every blob look unreferenced and flag the whole table as +/// `refcount_mismatch`. +fn chunk_page_sql(registry: &BlobReferenceRegistry) -> String { + let expected = registry.ref_count_expr(RefLevel::Chunk, "b.hash"); + assert!( + expected != "0", + "no chunk-level blob reference source registered: every blob would \ + appear unreferenced" + ); + + format!( + "SELECT + b.hash AS hash, + b.size AS size, + b.ref_count AS ref_count, + b.orphaned_at AS orphaned_at, + ({expected})::bigint AS actual_ref_count + FROM storage.blobs b + WHERE ($1::text IS NULL OR b.hash > $1) + ORDER BY b.hash + LIMIT $2" + ) +} + +/// Per-row corrective UPDATE for `storage.blobs.ref_count`, targeting +/// one blob by `hash`. Uses the SAME registry-derived expression as +/// [`chunk_page_sql`] so detection and repair agree on "actual" by +/// construction. A future `RefLevel::Chunk` ref source added to the +/// registry flows into both queries with no code change here. +/// +/// The `<> (subquery)` guard makes the UPDATE a no-op when the value +/// is already correct — idempotent under concurrent-repair races and +/// under retry. The subquery re-reads inside the same statement, so a +/// concurrent write between page fetch and this UPDATE can't leave a +/// stale value. +fn chunk_repair_sql(registry: &BlobReferenceRegistry) -> String { + let expected = registry.ref_count_expr(RefLevel::Chunk, "b.hash"); + format!( + "UPDATE storage.blobs b + SET ref_count = ({expected})::bigint + WHERE b.hash = $1 + AND b.ref_count <> ({expected})::bigint" + ) } impl BlobsConsistencyCheck { - pub fn new( - pool: Arc, - backend: Arc, - storage_entries: Vec, - storage_path_fallback: PathBuf, - ) -> Self { + pub fn new(pool: Arc, reference_registry: Arc) -> Self { Self { pool, - backend, - storage_entries, - storage_path_fallback, + chunk_page_sql: chunk_page_sql(&reference_registry), + chunk_repair_sql: chunk_repair_sql(&reference_registry), } } @@ -148,7 +217,13 @@ struct BlobRow { hash: String, size: i64, ref_count: i32, - created_at: DateTime, + /// Wall-clock instant this row hit `ref_count = 0` and became + /// eligible for GC. `NULL` for pre-migration rows or write-paths + /// that never stamped it — those the GC treats as immediately + /// reap-able (see `dedup_service.rs` phase-2 predicate), so the + /// stall check ignores them too: without a stamp we cannot say + /// how long a row has been sitting. + orphaned_at: Option>, /// Real reference count derived from the actual references — /// files' whole-file `blob_hash` PLUS every chunk hash across /// `storage.chunk_manifests`. Compared to `ref_count` (the @@ -162,6 +237,36 @@ impl RecoverableJobHandler for BlobsConsistencyCheck { BLOBS_CONSISTENCY_JOB_NAME } + fn description(&self) -> &'static str { + "Walks storage.blobs and checks two ref-counting invariants \ + dedup_gc relies on. First: refcount_mismatch — the stored \ + ref_count disagrees with the references that actually exist \ + (under-count lets GC reap a live blob, over-count pins a dead \ + one); repairable via ?repair=true. Second: blob_orphan_stalled \ + — the row satisfies every reap predicate GC uses but is still \ + present past 24× GC's grace, meaning the GC pipeline itself is \ + stuck (worker crashed, backend delete hanging, ghost referrer \ + being recreated); discovery-only, because a one-click repair \ + would hide the root cause the operator needs to fix — after \ + diagnosis, POST /api/admin/jobs/dedup_gc/trigger?force=true \ + drains the backlog. Database only — cheap and safe to run at \ + any time. Missing, orphaned or corrupted bytes are \ + backend_consistency's job." + } + + fn mutates(&self) -> Mutates { + Mutates::OnRepairOnly + } + + fn repair_description(&self) -> Option<&'static str> { + Some( + "Rewrites drifted ref_count values to the recomputed truth. \ + Does not delete blobs or resurrect missing bytes — an \ + over-counted blob simply becomes eligible for the next \ + dedup_gc sweep.", + ) + } + /// Definitive count. `storage.blobs` PK scan is index-only; /// even at millions of rows it's sub-second on modern PG. async fn count_total(&self) -> Option { @@ -188,78 +293,12 @@ impl RecoverableJobHandler for BlobsConsistencyCheck { args: &JobRunArgs, resume_cursor: Option>, ) -> RunOutcome { - // Resolve the backend to probe. Two paths, mirroring the - // Fresh/Resumed split the backend_migration handler uses: + // No backend is resolved here, and `?storage=` is inert: + // this tenant reads nothing but the database. Everything physical + // — existence, orphan bytes, bit-rot — belongs to + // `backend_consistency`, which finds it in one enumeration pass + // instead of one probe per row. // - // * Fresh + args.storage=Some — probe that named entry - // instead of the live backend. Stamp probed_storage in - // params so a mid-audit restart resumes against the same - // entry without re-input. - // * Fresh + args.storage=None — probe the live backend - // (today's default; audit of what the app is actually - // using). - // * Resumed — read probed_storage from params; None means - // the original run was against the live backend. - let is_fresh = resume_cursor.is_none(); - let probed_storage: Option = if is_fresh { - let name = args.storage.clone(); - if let Some(n) = &name - && let Err(e) = store.set_string_param(PROBED_STORAGE_PARAM, n).await - { - return RunOutcome::Failed { - message: format!("persist {PROBED_STORAGE_PARAM} to params: {e}"), - }; - } - name - } else { - match store.get_string_param(PROBED_STORAGE_PARAM).await { - Ok(v) => v, - Err(e) => { - return RunOutcome::Failed { - message: format!("read {PROBED_STORAGE_PARAM} from params: {e}"), - }; - } - } - }; - let backend: Arc = match &probed_storage { - None => self.backend.clone(), - Some(name) => match self.storage_entries.iter().find(|e| &e.name == name) { - Some(entry) => build_entry_backend(entry, &self.storage_path_fallback), - None => { - let available = if self.storage_entries.is_empty() { - "(none)".to_string() - } else { - self.storage_entries - .iter() - .map(|e| e.name.as_str()) - .collect::>() - .join(", ") - }; - return RunOutcome::Failed { - message: format!( - "storage entry `{name}` not found in OXICLOUD_STORAGE_ENTRIES. \ - Available: [{available}]" - ), - }; - } - }, - }; - if let Err(e) = backend.initialize().await { - return RunOutcome::Failed { - message: format!("probed backend init: {e}"), - }; - } - if let Some(name) = &probed_storage { - tracing::info!( - target: "audit", - event = "blobs_consistency.probe_scoped", - run_id = %store.run_id(), - probed_storage = %name, - "blobs_consistency probing entry `{name}` (via ?storage=) instead of \ - live backend" - ); - } - // Snapshot "is this a Fresh run?" BEFORE the resume_cursor // match consumes it — otherwise the `is_none()` check later // borrows a partially-moved value. Fresh = no cursor bytes @@ -286,55 +325,52 @@ impl RecoverableJobHandler for BlobsConsistencyCheck { // stats.finding_count — actual persistence happens in // `record_finding` on each emission). let mut finding_count = 0u64; + // Only touched when `args.repair == true`. Symmetric with + // `manifests_consistency`; reported in completion log + + // `extra_stats` so operators see "found N, fixed M" in one line. + let mut repaired_count = 0u64; + // Stall-check finding counter. Reported alongside + // `finding_count` (which covers refcount findings) so the + // completion line separates the two invariant classes. + let mut stalled_count = 0u64; - // Deep mode is a per-run flag with two consumers: - // 1. This handler — decides whether to re-hash bytes. - // 2. The admin panel — needs to display whether the run - // was deep so operators know what the scan actually - // verified. - // - // On a Fresh run we take it from `deep` (the trigger - // endpoint stamps `?deep=true` onto the args) and stash it - // in `params.deep` so: - // * Resume picks up the same mode (would previously become - // non-deep on Resume — a Paused deep scan silently lost - // its `deep` intent). - // * The admin panel run-detail view can render - // `params.deep = "true"` alongside `target_name`, - // `progress_kind`, etc. - // - // Persist BEFORE the walk so a mid-fresh-batch crash still - // leaves a Paused row with the right mode marker. - let deep = if is_fresh { - let deep = args.deep; - let v = if deep { "true" } else { "false" }; - if let Err(e) = store.set_string_param("deep", v).await { + // `?deep=true` is not handled here. Re-reading and re-hashing + // bytes is backend work end to end, so it moved to + // `backend_consistency`, where the merge-join already holds the + // matched key pairs worth verifying. A deep flag on this tenant + // would be a flag with nothing to do. + + // Repair mode persisted to `params.repair` so the admin run-detail + // view can display it. Fresh persists what the trigger asked for; + // Resume reads back so a paused repair scan stays a repair + // scan (a mid-scan crash mustn't silently downgrade to + // discovery-only for the remaining rows). + let repair = if is_fresh { + let v = if args.repair { "true" } else { "false" }; + if let Err(e) = store.set_string_param("repair", v).await { return RunOutcome::Failed { - message: format!("failed to persist deep flag to params: {e}"), + message: format!("failed to persist repair flag to params: {e}"), }; } - deep + args.repair } else { - // Resumed run — read the persisted flag. Default to - // false (fast mode) if the row is a pre-K3.5 Paused - // scan without the param stashed. - match store.get_string_param("deep").await { + match store.get_string_param("repair").await { Ok(Some(v)) => v == "true", Ok(None) => false, Err(e) => { return RunOutcome::Failed { - message: format!("read `deep` from params: {e}"), + message: format!("read `repair` from params: {e}"), }; } } }; - if deep { + if repair { tracing::info!( target: "oxicloud::consistency", - event = "blobs_consistency.deep_mode_active", + event = "blobs_consistency.repair_mode_active", run_id = %store.run_id(), - "deep mode: re-reading + re-hashing every blob (bit-rot detection)" + "repair mode: refcount_mismatch findings will trigger corrective UPDATE" ); } @@ -347,6 +383,7 @@ impl RecoverableJobHandler for BlobsConsistencyCheck { event = "blobs_consistency.cancelled", run_id = %store.run_id(), finding_count = finding_count, + stalled_count = stalled_count, "blobs_consistency cancelled cooperatively, pausing" ); return RunOutcome::Paused { @@ -364,58 +401,14 @@ impl RecoverableJobHandler for BlobsConsistencyCheck { } } - // Fetch the next batch. Per-row `actual_ref_count` - // computed inline via correlated subqueries — one for - // legacy whole-file references (`files.blob_hash`), one - // for CDC chunk references (`chunk_manifests.chunk_hashes`). - // GIN index on `chunk_hashes` (migration - // 20260628000000_delta_upload_gin_index) makes the - // `= ANY(chunk_hashes)` probe cheap. - // `storage.blobs.ref_count` semantics — what the invariant - // dedup_service maintains actually is: - // - // ref_count = (number of chunk_manifests whose - // chunk_hashes[] contains this hash) - // + (number of files.blob_hash pointing at - // this hash on the LEGACY whole-file path - // — i.e. files with NO manifest for their - // blob_hash) - // - // Naively `COUNT(files) + COUNT(manifests referring)` - // double-counts single-chunk CDC files: for a file whose - // whole-file hash == its single chunk's hash (any file - // small enough to fit in one CDC chunk — under ~256 KB - // average), the file appears BOTH in `files.blob_hash` - // AND in the manifest's `chunk_hashes[]`. The `NOT - // EXISTS` clause below excludes CDC-path files from the - // legacy count so the two terms don't overlap. - let rows: Vec = match sqlx::query_as( - r#" - SELECT - b.hash AS hash, - b.size AS size, - b.ref_count AS ref_count, - b.created_at AS created_at, - ( - (SELECT COUNT(*) FROM storage.files f - WHERE f.blob_hash = b.hash - AND NOT EXISTS ( - SELECT 1 FROM storage.chunk_manifests m - WHERE m.file_hash = f.blob_hash - )) - + (SELECT COUNT(*) FROM storage.chunk_manifests m - WHERE b.hash = ANY(m.chunk_hashes)) - )::bigint AS actual_ref_count - FROM storage.blobs b - WHERE ($1::text IS NULL OR b.hash > $1) - ORDER BY b.hash - LIMIT $2 - "#, - ) - .bind(cursor.as_deref()) - .bind(BATCH_SIZE) - .fetch_all(self.pool.as_ref()) - .await + // Fetch the next batch. `actual_ref_count` is summed from the + // registered reference sources — see `chunk_page_sql`, which + // documents the invariant and the single-chunk double-count trap. + let rows: Vec = match sqlx::query_as(&self.chunk_page_sql) + .bind(cursor.as_deref()) + .bind(BATCH_SIZE) + .fetch_all(self.pool.as_ref()) + .await { Ok(r) => r, Err(e) => { @@ -431,168 +424,179 @@ impl RecoverableJobHandler for BlobsConsistencyCheck { event = "blobs_consistency.completed", run_id = %store.run_id(), finding_count = finding_count, - deep = deep, - "blobs_consistency completed with {} finding(s)", - finding_count + repaired_count = repaired_count, + stalled_count = stalled_count, + repair_requested = repair, + "blobs_consistency completed with {} refcount finding(s), \ + {} repaired, {} stalled", + finding_count, + repaired_count, + stalled_count, ); - return RunOutcome::completed(); + return RunOutcome::completed_with(serde_json::json!({ + "repair_requested": repair, + "repaired_count": repaired_count, + "stalled_count": stalled_count, + })); } - let grace_cutoff = Utc::now() - CREATE_GRACE; - + // No grace window here any more. It existed to keep the + // physical probe from flagging a blob whose bytes had landed + // but whose row hadn't — a write-path race this tenant no + // longer looks at. The refcount comparison reads one + // consistent DB snapshot, so there is nothing to wait for. for row in &rows { - // (1) refcount_mismatch — content-safe check, cheap, - // always runs. Emitted BEFORE the physical probe so - // a broken-and-miscounted blob shows both findings. - if row.ref_count as i64 != row.actual_ref_count { - finding_count += 1; - let affected = affected_files(self.pool.as_ref(), &row.hash).await; - record_or_log( - store, - BLOBS_CONSISTENCY_JOB_NAME, - "refcount_mismatch", - "inconsistent", - None, // hash isn't a UUID; resource identifier lives in detail - serde_json::json!({ - "hash": row.hash, - "stored": row.ref_count, - "actual": row.actual_ref_count, - "delta": row.actual_ref_count - row.ref_count as i64, - "size": row.size, - "affected_files": affected, - }), - ) - .await; - } - - // Skip physical probes for rows within the write - // grace window — writes-in-flight would false-positive. - if row.created_at > grace_cutoff { - continue; - } - - // (2) blob_missing_from_backend — normal mode - // physical existence probe. Fails-open on backend - // error (log + skip): a transient S3 network blip - // shouldn't produce a flood of false data_loss - // findings. - let exists = match backend.blob_exists(&row.hash).await { - Ok(v) => v, - Err(e) => { - tracing::warn!( - target: "oxicloud::consistency", - event = "blobs_consistency.blob_exists_error", - run_id = %store.run_id(), - hash = %row.hash, - error = %e, - "blob_exists probe failed; skipping this row" - ); - continue; - } - }; - - if !exists { - finding_count += 1; - let affected = affected_files(self.pool.as_ref(), &row.hash).await; - record_or_log( - store, - BLOBS_CONSISTENCY_JOB_NAME, - "blob_missing_from_backend", - "data_loss", - None, - serde_json::json!({ - "hash": row.hash, - "size": row.size, - "ref_count": row.ref_count, - "affected_files": affected, - }), - ) - .await; - // No point re-hashing bytes that aren't there. - continue; - } - - // (3) blob_corrupted — DEEP MODE only. Read the - // whole blob, recompute BLAKE3, compare to the hash - // it's indexed under. Any mismatch = silent bit-rot. - // - // Finding fields: - // * `hash` — expected hash (the key the blob is - // indexed under in `storage.blobs`). - // * `computed_hash` — what BLAKE3 of the current - // bytes actually produces. Diagnostic: a - // one-bit flip vs a truncation vs a whole-file - // swap all leave distinctive signatures. - // `expected_hash` was NOT reused as a name to - // avoid mistaking it for "the hash we expect to - // see on disk (i.e. what will fix this)". - if deep { - match recompute_hash(backend.as_ref(), &row.hash).await { - Ok(computed_hash) if computed_hash == row.hash => {} - Ok(computed_hash) => { - finding_count += 1; + if row.ref_count as i64 == row.actual_ref_count { + // No drift. Check for stall — orthogonal condition, + // only meaningful when the counter is CORRECT: if + // drift existed, the drift IS the reason the GC + // hasn't taken this row, and firing stall on top + // would mislead the operator into hunting a + // GC-pipeline issue that isn't there. Fix the + // counter → the row becomes eligible on the next + // sweep. Only when counter == actual == 0 AND the + // row has been sitting past `STALL_GRACE_SECS` is + // this a genuine "the GC should have taken this + // and hasn't" signal. + if row.actual_ref_count == 0 + && let Some(orphaned_at) = row.orphaned_at + { + let stalled_secs = (Utc::now() - orphaned_at).num_seconds(); + if stalled_secs > STALL_GRACE_SECS { + stalled_count += 1; let affected = affected_files(self.pool.as_ref(), &row.hash).await; + let detail = serde_json::json!({ + "hash": row.hash, + "size": row.size, + "ref_count": row.ref_count, + "orphaned_at": orphaned_at, + "stalled_for_secs": stalled_secs, + "stall_grace_secs": STALL_GRACE_SECS, + "affected_files": affected, + // Inline hint the admin UI can render + // on click. Not repaired here (see + // module doc) — after operator has + // diagnosed the root cause (worker + // wedged, backend hang, ghost row), + // this is the one-shot to drain the + // backlog. + "remediation_hint": "Investigate why dedup_gc has not reaped this row \ + (worker running? advisory-lock contention? backend delete hanging? \ + ghost chunk_manifests/storage.files row?), then \ + POST /api/admin/jobs/dedup_gc/trigger?force=true to drain the backlog.", + }); record_or_log( store, BLOBS_CONSISTENCY_JOB_NAME, - "blob_corrupted", - "data_loss", - None, - serde_json::json!({ - "hash": row.hash, - "computed_hash": computed_hash, - "size": row.size, - "ref_count": row.ref_count, - "affected_files": affected, - }), + // Stable machine-readable kind — the + // admin UI and log-aggregator queries + // key off this string. Do not rename. + "blob_orphan_stalled", + // "anomaly" — surprising state worth + // surfacing, no direct data impact. + // The bytes are safe; their persistence + // past grace means the reap pipeline + // needs attention. + "anomaly", + None, // hash isn't a UUID; identifier lives in detail + detail, ) .await; } + } + continue; + } + finding_count += 1; + let affected = affected_files(self.pool.as_ref(), &row.hash).await; + let detail = serde_json::json!({ + "hash": row.hash, + "stored": row.ref_count, + "actual": row.actual_ref_count, + "delta": row.actual_ref_count - row.ref_count as i64, + "size": row.size, + "affected_files": affected, + }); + + // Repair pass — content-safe corrective UPDATE. Sets + // `stored` to the value the auditor formula would + // compute at UPDATE time (subquery matches + // `chunk_page_sql`'s `actual_ref_count`), so a + // concurrent write between our page fetch and this + // UPDATE can't leave a stale value — the subquery + // re-reads inside the same statement. The `<>` + // guard makes the UPDATE a no-op if the value is + // already correct, so this is idempotent under retry. + // + // `self.chunk_repair_sql` is built once at construction + // from the same `BlobReferenceRegistry` as the page + // query — detection and repair use identical formulas + // by construction. See `chunk_repair_sql`. + // + // Attempt repair FIRST, then record the finding with + // severity/kind reflecting the final state: + // * repair succeeded → severity "info", kind "refcount_repaired" + // * repair no-op → severity "info", kind "refcount_resolved" + // * repair failed → severity "inconsistent", kind "refcount_mismatch" + // * no repair requested → severity "inconsistent", kind "refcount_mismatch" + // + // Parallels the WARN-then-INFO sequence in logs: an + // unresolved drift raises attention ("inconsistent"), + // a repaired one records the fix at info level without + // inflating the "needs action" tally the outcome UI + // shows. The detail JSON still carries `stored/actual/ + // delta/affected_files` so the audit trail is complete + // either way. + let (kind, severity) = if repair { + match sqlx::query(&self.chunk_repair_sql) + .bind(&row.hash) + .execute(self.pool.as_ref()) + .await + { + Ok(res) if res.rows_affected() > 0 => { + repaired_count += 1; + tracing::info!( + target: "audit", + event = "blobs_consistency.repaired", + run_id = %store.run_id(), + hash = %row.hash, + stored_was = row.ref_count, + actual = row.actual_ref_count, + "🩹 blob ref_count repaired" + ); + ("refcount_repaired", "info") + } + Ok(_) => { + // Row not touched — either another + // concurrent repair fixed it first, or + // drift healed between page fetch and + // UPDATE. Current state correct — info. + ("refcount_resolved", "info") + } Err(e) => { - // Blob can't be read at all — record as - // `blob_unreadable`. Distinct from - // `blob_corrupted` (hash mismatch = we - // can read but content differs): here - // we can't get bytes out to hash. Common - // causes: decrypt failure (missing key), - // network glitch on S3/Azure, missing - // file on Local, permission error. - // - // Recorded as `data_loss` because from - // the file's perspective the outcome is - // the same as corruption: content is - // inaccessible. Admins triage the error - // string to distinguish transient - // (retry-safe) from permanent (needs - // key recovery or blob replacement). - finding_count += 1; - let affected = affected_files(self.pool.as_ref(), &row.hash).await; - record_or_log( - store, - BLOBS_CONSISTENCY_JOB_NAME, - "blob_unreadable", - "data_loss", - None, - serde_json::json!({ - "hash": row.hash, - "size": row.size, - "ref_count": row.ref_count, - "affected_files": affected, - "error": e.to_string(), - }), - ) - .await; tracing::warn!( target: "oxicloud::consistency", - event = "blobs_consistency.blob_unreadable", + event = "blobs_consistency.repair_failed", run_id = %store.run_id(), hash = %row.hash, error = %e, - "🚨 blob unreadable in deep mode — recorded finding, continuing" + "blob ref_count repair UPDATE failed — finding stays" ); + ("refcount_mismatch", "inconsistent") } } - } + } else { + ("refcount_mismatch", "inconsistent") + }; + + record_or_log( + store, + BLOBS_CONSISTENCY_JOB_NAME, + kind, + severity, + None, // hash isn't a UUID; resource identifier lives in detail + detail, + ) + .await; } // Advance cursor + checkpoint. @@ -611,75 +615,108 @@ impl RecoverableJobHandler for BlobsConsistencyCheck { event = "blobs_consistency.completed", run_id = %store.run_id(), finding_count = finding_count, - deep = deep, - "blobs_consistency completed with {} finding(s)", - finding_count + repaired_count = repaired_count, + repair_requested = repair, + "blobs_consistency completed with {} finding(s), {} repaired", + finding_count, + repaired_count ); - return RunOutcome::completed(); + return RunOutcome::completed_with(serde_json::json!({ + "repair_requested": repair, + "repaired_count": repaired_count, + })); } } } } -/// Sample of file names that reference this blob — either directly -/// (`files.blob_hash = $hash`, legacy pre-CDC) or transitively via a -/// manifest (`chunk_hashes @> ARRAY[$hash]`, post-CDC dominant path). -/// Capped so a chunk shared by 10 000 files doesn't blow up the -/// finding detail JSON. Order is arbitrary — sampling for -/// diagnosis, not enumeration. -async fn affected_files(pool: &PgPool, hash: &str) -> Vec { - let rows: Vec<(String,)> = sqlx::query_as( - r#" - SELECT DISTINCT f.name - FROM storage.files f - WHERE f.blob_hash = $1 - OR EXISTS ( - SELECT 1 FROM storage.chunk_manifests m - WHERE m.file_hash = f.blob_hash - AND $1 = ANY(m.chunk_hashes) - ) - LIMIT $2 - "#, - ) - .bind(hash) - .bind(AFFECTED_FILES_SAMPLE) - .fetch_all(pool) - .await - .unwrap_or_default(); - rows.into_iter().map(|(n,)| n).collect() -} +#[cfg(test)] +mod tests { + use super::*; -/// Deep-mode helper — read the blob from the backend and recompute -/// its BLAKE3 hash. Returns `Ok(true)` when the recomputed hash -/// matches `expected_hash` (byte for byte), `Ok(false)` on mismatch -/// (bit-rot), `Err(_)` on any backend-side error (network blip, -/// permission issue) — callers log-and-skip errors since a transient -/// failure isn't a corruption signal. -/// Deep-mode helper — read the blob from the backend and recompute -/// its BLAKE3 hash. Returns the recomputed hex string; callers -/// compare against the expected hash themselves. Returning the -/// actual hash (not just a bool) lets the finding surface WHAT the -/// bytes now hash to, which is diagnostic gold: a specific one-bit -/// flip has a very different signature from a chunk-boundary -/// corruption or a truncated read. `Err(_)` on backend-side error -/// (network blip, permission issue) — callers log-and-skip since -/// transient failure isn't a corruption signal. -async fn recompute_hash( - backend: &dyn BlobStorageBackend, - expected_hash: &str, -) -> Result { - use crate::common::errors::DomainError; - use futures::StreamExt; - - let mut stream = backend.get_blob_stream(expected_hash).await?; - let mut hasher = blake3::Hasher::new(); - - while let Some(chunk) = stream.next().await { - let bytes = chunk.map_err(|e| { - DomainError::internal_error("BlobsConsistency", format!("stream read: {e}")) - })?; - hasher.update(&bytes); + fn default_registry() -> BlobReferenceRegistry { + let pool = Arc::new( + sqlx::pool::PoolOptions::::new() + .connect_lazy("postgres://invalid/invalid") + .expect("lazy pool never connects"), + ); + crate::infrastructure::repositories::pg::blob_reference_sources::built_in_registry(pool) } - Ok(hasher.finalize().to_hex().to_string()) + /// Golden test for the chunk-level recompute. Pins the statement + /// byte-for-byte because it is assembled from the registry rather than + /// written as a literal — the reviewer should read the SQL here. + /// + /// This expression must stay equal to what the query computed before the + /// registry existed: the legacy-files term guarded by `NOT EXISTS`, plus + /// the manifests-citing-this-chunk term. If a change makes those two + /// overlap, every single-chunk CDC file is counted twice and the whole + /// table reports `refcount_mismatch`. + #[tokio::test] + async fn chunk_page_statement_is_stable() { + let sql = chunk_page_sql(&default_registry()); + let expected = r#"SELECT + b.hash AS hash, + b.size AS size, + b.ref_count AS ref_count, + b.orphaned_at AS orphaned_at, + ((SELECT COUNT(*) FROM storage.files cnt_f + WHERE cnt_f.blob_hash = b.hash + AND NOT EXISTS ( + SELECT 1 FROM storage.chunk_manifests cnt_m + WHERE cnt_m.file_hash = cnt_f.blob_hash + )) + + (SELECT COUNT(*) FROM storage.chunk_manifests cnt_m + WHERE b.hash = ANY(cnt_m.chunk_hashes)))::bigint AS actual_ref_count + FROM storage.blobs b + WHERE ($1::text IS NULL OR b.hash > $1) + ORDER BY b.hash + LIMIT $2"#; + assert_eq!(sql, expected, "chunk page statement changed:\n{sql}"); + } + + /// With no chunk-level source every blob would look unreferenced and the + /// sweep would report the entire table as `refcount_mismatch`. Refuse to + /// build the statement instead. + #[test] + #[should_panic(expected = "no chunk-level blob reference source")] + fn empty_registry_refuses_to_build_page_statement() { + let _ = chunk_page_sql(&BlobReferenceRegistry::new()); + } + + /// Golden test — the repair statement is assembled from the same + /// registry as `chunk_page_sql`, so pin it byte-for-byte too. If + /// the registry ever changes what it produces at + /// `RefLevel::Chunk`, BOTH this test and + /// `chunk_page_statement_is_stable` above break together — an + /// operator using `?repair=true` shouldn't see the detection + /// formula report drift the repair formula can't clear. + /// + /// Ships the two-term formula (`storage.files` legacy-path count + + /// `storage.chunk_manifests` chunk-membership count) twice — once + /// in SET, once in the `<>` guard. Both must stay identical so the + /// guard is meaningful. + #[tokio::test] + async fn chunk_repair_statement_is_stable() { + let sql = chunk_repair_sql(&default_registry()); + let expected = r#"UPDATE storage.blobs b + SET ref_count = ((SELECT COUNT(*) FROM storage.files cnt_f + WHERE cnt_f.blob_hash = b.hash + AND NOT EXISTS ( + SELECT 1 FROM storage.chunk_manifests cnt_m + WHERE cnt_m.file_hash = cnt_f.blob_hash + )) + + (SELECT COUNT(*) FROM storage.chunk_manifests cnt_m + WHERE b.hash = ANY(cnt_m.chunk_hashes)))::bigint + WHERE b.hash = $1 + AND b.ref_count <> ((SELECT COUNT(*) FROM storage.files cnt_f + WHERE cnt_f.blob_hash = b.hash + AND NOT EXISTS ( + SELECT 1 FROM storage.chunk_manifests cnt_m + WHERE cnt_m.file_hash = cnt_f.blob_hash + )) + + (SELECT COUNT(*) FROM storage.chunk_manifests cnt_m + WHERE b.hash = ANY(cnt_m.chunk_hashes)))::bigint"#; + assert_eq!(sql, expected, "chunk repair statement changed:\n{sql}"); + } } diff --git a/src/infrastructure/services/consistency_batch_service.rs b/src/infrastructure/services/consistency_batch_service.rs index ad174da1..d4287375 100644 --- a/src/infrastructure/services/consistency_batch_service.rs +++ b/src/infrastructure/services/consistency_batch_service.rs @@ -59,7 +59,7 @@ use std::sync::{Arc, Weak}; use async_trait::async_trait; use serde_json::json; -use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs}; +use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs, Mutates}; pub const CONSISTENCY_BATCH_JOB_NAME: &str = "consistency_batch"; @@ -90,6 +90,28 @@ impl JobHandler for ConsistencyBatch { CONSISTENCY_BATCH_JOB_NAME } + fn description(&self) -> &'static str { + "Runs every registered consistency check in sequence — one click \ + for 'check everything'. New tenants are picked up automatically \ + by name, so nothing needs updating here when one is added. Flags \ + are forwarded to each sub-job." + } + + /// Read-only on a plain run because every tenant it dispatches is, but + /// `?repair=true` reaches whichever of them act on it — so the batch + /// inherits the strongest mode any sub-job can be put into. + fn mutates(&self) -> Mutates { + Mutates::OnRepairOnly + } + + fn repair_description(&self) -> Option<&'static str> { + Some( + "Forwards ?repair=true to every sub-check, so the ones that \ + support it fix what they find (today: refcount drift on blobs \ + and manifests) instead of only reporting it.", + ) + } + async fn run(&self, args: &JobRunArgs) -> JobOutcome { // Upgrade the Weak. Only fails if the registry has been // dropped — which can only happen during process shutdown, @@ -178,6 +200,7 @@ impl JobHandler for ConsistencyBatch { "per_check": per_check, "deep": args.deep, "force": args.force, + "repair": args.repair, "ok": ok_count, "err": err_count, }), diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index c8b84fa3..f5faab14 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -55,6 +55,7 @@ use std::sync::Arc; use tokio_util::io::StreamReader; use crate::application::ports::blob_lifecycle::BlobLifecycleHook; +use crate::application::ports::blob_reference_ports::{BlobReferenceRegistry, RefLevel}; use crate::application::ports::blob_storage_ports::BlobStorageBackend; use crate::application::ports::dedup_ports::{ BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto, @@ -424,6 +425,147 @@ async fn populate_integrity_blob_sizes<'a>( IntegrityBlobSizes { hashes, sizes } } +/// Build the manifest reap statement from the registered reference sources. +/// +/// **A manifest is collectible when, and only when, no registered source +/// references it.** The reference registry is the sole authority; `ref_count` +/// does not appear in this predicate at all. +/// +/// # Why `ref_count` was removed from it +/// +/// This used to read `ref_count <= 0 OR `. Each arm had a +/// purpose — the single-file delete path decrements the counter via +/// `cleanup_if_orphaned`, while bulk paths (user cascade, `empty_trash`) only +/// fire the `storage.blobs` trigger and leave the counter untouched — so the +/// disjunction looked like belt and braces. +/// +/// It was the opposite. With `OR`, **either signal alone deletes**, so a +/// counter that under-reports does not merely report a wrong number: it makes +/// live content collectible, and the registry that knows better is never +/// consulted because the first arm already matched. That is not hypothetical. +/// `storage.copy_folder_tree` used to take references with +/// `UPDATE storage.blobs … WHERE hash = blob_hash`, which matches nothing for +/// a CDC file — whose `blob_hash` names a manifest, not a chunk — so it took +/// no reference at all. Copy a folder, delete the original, and the copy's +/// bytes were reaped. +/// +/// Dropping the counter arm loses no coverage, because the single-file path +/// deletes the `storage.files` row too, which makes the row unreferenced +/// anyway. And it costs no performance: under `OR`, Postgres had to evaluate +/// the `EXISTS` union for every row whose `ref_count` was above zero — which +/// on a healthy install is nearly all of them — so the expensive predicate was +/// already running unconditionally. +/// +/// What it does change: a counter stuck *high* with no referrers left is no +/// longer reaped here. That is the bulk-delete residue, and it now belongs to +/// the manifest-level refcount recompute (`docs/plan/derived-blobs.md`, +/// coverage matrix row 7) — a counter being wrong is a job for the thing that +/// reconciles counters, not for the thing that deletes data. +/// +/// The predicate is registry-driven rather than naming `storage.files` +/// directly, so a new referring table — thumbnails via +/// `storage.content_derived_blobs`, previews via +/// `storage.file_attached_blobs` — is covered by registering its source. +/// Hardcoded, each new table would have had its manifests reaped on the next +/// sweep despite a correct `ref_count`. +/// +/// Pinned by `gc_reference_authority_integration_tests`. +/// +/// # Panics +/// +/// If no source contributes at [`RefLevel::Manifest`]. That is a wiring bug, +/// and it must be loud: with no source, "nothing references it" is vacuously +/// true for every row and this statement would delete every manifest in the +/// database. `DedupService::new` always registers `FilesReferenceSource`, so +/// the only way to reach this is to pass a deliberately empty registry. +/// Build the chunk/blob reap statement (GC phase 2) from the registered +/// reference sources. +/// +/// Unlike [`manifest_reap_sql`], the registry predicate here is **added to** +/// the hardcoded guards rather than replacing them. That asymmetry is +/// deliberate and the reason this was not a mechanical swap. +/// +/// `no_reference_predicate` is built from fragments designed for *counting*, +/// and `FilesReferenceSource`'s chunk-level fragment deliberately excludes +/// files whose `blob_hash` has a manifest — otherwise a single-chunk blob, +/// where the file hash and its lone chunk hash are the same BLAKE3, would be +/// counted at both levels. Correct for a recompute; too narrow for a reap +/// guard. A `storage.blobs` row keyed by a MULTI-chunk file's hash — which +/// exists transiently while `rechunk` migrates a legacy blob, and is not a +/// member of its own manifest's `chunk_hashes` — would satisfy the registry's +/// "unreferenced" test while a live `storage.files` row still points at it. +/// Swapping the guards out would have reaped it mid-migration. +/// +/// So the statement keeps `NOT EXISTS (manifest lists it as a chunk)` and +/// `NOT EXISTS (any file points at it)`, and ANDs the registry predicate on +/// top. Adding a conjunct can only ever spare more rows, never reap more, so +/// this cannot regress; what it buys is that a future source contributing at +/// [`RefLevel::Chunk`] is honoured automatically instead of being silently +/// missed — the same failure that made Phase 1's hardcoded cross-check +/// dangerous. +/// +/// Today the registry adds nothing operationally: +/// `content_derived_blobs` and `file_attached_blobs` both return `None` at +/// `RefLevel::Chunk`, so its union is exactly manifests + legacy files. The +/// point is what happens when that stops being true. +/// +/// `$1` is the batch limit, `$2` the grace window in seconds. +/// +/// # Panics +/// +/// If no source contributes at [`RefLevel::Chunk`]. Same reasoning as +/// [`manifest_reap_sql`]: a missing predicate must be loud rather than +/// silently degrading to "nothing references anything". +fn blob_reap_sql(registry: &BlobReferenceRegistry) -> String { + let unreferenced = registry + .no_reference_predicate(RefLevel::Chunk, "b.hash") + .expect( + "no chunk-level blob reference source registered: the reap \ + predicate would lose its registry cross-check", + ); + + format!( + "DELETE FROM storage.blobs + WHERE ctid = ANY( + SELECT b.ctid FROM storage.blobs b + WHERE b.ref_count <= 0 + AND (b.orphaned_at IS NULL + OR b.orphaned_at < now() - ($2::int * interval '1 second')) + AND NOT EXISTS ( + SELECT 1 FROM storage.chunk_manifests m + WHERE m.chunk_hashes @> ARRAY[b.hash::text] + ) + AND NOT EXISTS ( + SELECT 1 FROM storage.files f + WHERE f.blob_hash = b.hash + ) + AND {unreferenced} + LIMIT $1 + ) + RETURNING hash, size" + ) +} + +fn manifest_reap_sql(registry: &BlobReferenceRegistry) -> String { + let orphaned = registry + .no_reference_predicate(RefLevel::Manifest, "m.file_hash") + .expect( + "no manifest-level blob reference source registered: the reap \ + predicate would match every manifest", + ); + + format!( + "DELETE FROM storage.chunk_manifests + WHERE ctid = ANY( + SELECT ctid + FROM storage.chunk_manifests m + WHERE {orphaned} + LIMIT $1 + ) + RETURNING file_hash, chunk_hashes, total_size" + ) +} + pub struct DedupService { /// Pluggable blob storage backend (local FS, S3, …). backend: Arc, @@ -442,6 +584,20 @@ pub struct DedupService { /// seen immediately), weight-bounded (a manifest is ~72 B per chunk), /// short TTL so GC'd manifests age out fast (benches/MANIFEST-CACHE.md). manifest_cache: moka::future::Cache>, + /// Every table that holds blob references, so GC agrees with the + /// consistency jobs on what "referenced" means. Defaults to the two + /// built-in sources; DI replaces it once more tables exist. Never + /// optional — an empty registry would make "nothing references it" + /// vacuously true and the manifest sweep would reap everything. + reference_registry: Arc, + /// The manifest reap statement, built once from `reference_registry`. + /// Kept as a field so `garbage_collect` runs a fixed statement rather + /// than assembling SQL inside a delete loop — see `manifest_reap_sql`. + manifest_reap_sql: String, + /// The chunk/blob reap statement (GC phase 2), same treatment — see + /// [`blob_reap_sql`], including why its registry predicate is additive + /// rather than a replacement for the hardcoded guards. + blob_reap_sql: String, } impl DedupService { @@ -455,15 +611,32 @@ impl DedupService { pool: Arc, maintenance_pool: Arc, ) -> Self { + let registry = Arc::new(Self::default_reference_registry(pool.clone())); Self { backend, pool, maintenance_pool, blob_lifecycle: None, manifest_cache: Self::build_manifest_cache(), + reference_registry: registry.clone(), + manifest_reap_sql: manifest_reap_sql(®istry), + blob_reap_sql: blob_reap_sql(®istry), } } + /// Every built-in blob-reference source, in one place. + /// + /// This is THE definition of "what references a blob" — DI does not + /// assemble its own, it reads this one back via + /// [`Self::reference_registry`] and hands it to the consistency jobs, so + /// GC and the sweeps cannot disagree. Keeping it as the construction + /// default also means every path — including tests — has a + /// manifest-level source, so the reap predicate can never degenerate to + /// "nothing references anything". + fn default_reference_registry(pool: Arc) -> BlobReferenceRegistry { + crate::infrastructure::repositories::pg::blob_reference_sources::built_in_registry(pool) + } + /// See the `manifest_cache` field docs. Weight ≈ real heap bytes of one /// entry; 32 MiB cap ≈ tens of thousands of typical (sub-1 GB) files. fn build_manifest_cache() -> moka::future::Cache> { @@ -476,6 +649,338 @@ impl DedupService { .build() } + /// Registers the blob-reference registry used by the manifest reap + /// predicate. Without it `garbage_collect` skips manifest collection + /// entirely — see `docs/plan/derived-blobs.md`. + pub fn with_reference_registry(mut self, registry: Arc) -> Self { + self.manifest_reap_sql = manifest_reap_sql(®istry); + self.blob_reap_sql = blob_reap_sql(®istry); + self.reference_registry = registry; + self + } + + /// Store a server-derived artifact and record the mapping from the + /// content it was derived from. + /// + /// One call does the whole contract, so no caller has to remember the + /// accounting: + /// + /// 1. writes the bytes through the normal CDC path — derived blobs get + /// the same backend, encryption, migration and rotation as any other + /// content, and `store_from_stream` takes exactly one reference; + /// 2. records `(source_hash, kind, variant) -> blob_hash`; + /// 3. **releases that reference if the mapping already existed**, because + /// the row that would justify it is not ours — two instances racing + /// to render the same thumbnail must leave `ref_count` at 1, not 2. + /// + /// `bytes` is expected to be small (a thumbnail is 3-90 KB, below + /// `CDC_MIN_CHUNK`, so this is a single chunk). See + /// `docs/plan/derived-blobs.md`. + /// + /// Returns the derived blob hash. + /// Attach user-supplied bytes to a FILE — the file-keyed twin of + /// [`Self::store_derived_blob`]. + /// + /// Same storage path (the bytes are still content-addressed and still + /// deduplicated), different mapping: the row is keyed by `file_id`, so + /// two files holding identical attached bytes get two rows and two + /// references. Sharing the mapping is what must not happen — a + /// content-keyed client preview would let one user's upload be served + /// for another user's file. + /// + /// `ON CONFLICT … DO UPDATE`, unlike the derived twin: re-uploading a + /// preview for the same `(file_id, kind, variant)` is a deliberate + /// replacement, whereas a re-derived thumbnail is the same bytes again. + /// The reference held by the row being replaced is released. + pub async fn store_attached_blob( + &self, + file_id: &str, + kind: &str, + variant: &str, + content_type: &str, + bytes: Bytes, + uploaded_by: uuid::Uuid, + ) -> Result { + let stored = self + .store_from_stream( + stream::once(async move { Ok::(bytes) }), + Some(content_type.to_string()), + ) + .await?; + let attached_hash = stored.hash().to_string(); + + // Read the hash being superseded BEFORE upserting. + // + // It cannot come from `RETURNING`: PostgreSQL only permits `EXCLUDED` + // in the `SET` and `WHERE` of `DO UPDATE`, so a RETURNING clause + // comparing old against new is a syntax error — and one that surfaces + // only at runtime, where this method's best-effort caller swallows it + // into a warning while the sidecar keeps the feature looking healthy. + // + // The gap between this SELECT and the upsert is benign: losing the + // race leaves one stale reference, which the manifest recompute + // reports rather than anything being lost or served wrongly. + let previous: Option<(String,)> = sqlx::query_as( + "SELECT blob_hash FROM storage.file_attached_blobs + WHERE file_id = $1::uuid AND kind = $2 AND variant = $3", + ) + .bind(file_id) + .bind(kind) + .bind(variant) + .fetch_optional(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("Dedup", format!("read attached blob: {e}")))?; + + sqlx::query( + "INSERT INTO storage.file_attached_blobs + (file_id, kind, variant, blob_hash, content_type, uploaded_by) + VALUES ($1::uuid, $2, $3, $4, $5, $6) + ON CONFLICT (file_id, kind, variant) DO UPDATE + SET blob_hash = EXCLUDED.blob_hash, + content_type = EXCLUDED.content_type, + uploaded_by = EXCLUDED.uploaded_by, + created_at = now()", + ) + .bind(file_id) + .bind(kind) + .bind(variant) + .bind(&attached_hash) + .bind(content_type) + .bind(uploaded_by) + .execute(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("Dedup", format!("record attached blob: {e}")))?; + + // A replaced row's old blob loses its only reference from here. Not + // releasing it would pin those bytes forever — nothing else points at + // a superseded preview. + if let Some((old_hash,)) = previous + && old_hash != attached_hash + && let Err(e) = self.remove_reference(&old_hash).await + { + tracing::warn!( + target: "oxicloud::dedup", + error = %e, + "failed to release replaced attached-blob reference for {}", + &old_hash[..old_hash.len().min(12)], + ); + } + + Ok(attached_hash) + } + + /// Look up bytes attached to a file. File-keyed counterpart of + /// [`Self::find_derived_blob`]. + pub async fn find_attached_blob( + &self, + file_id: &str, + kind: &str, + variant: &str, + ) -> Option { + sqlx::query_as::<_, (String, String)>( + "SELECT blob_hash, content_type FROM storage.file_attached_blobs + WHERE file_id = $1::uuid AND kind = $2 AND variant = $3", + ) + .bind(file_id) + .bind(kind) + .bind(variant) + .fetch_optional(self.pool.as_ref()) + .await + .ok() + .flatten() + .map(|(blob_hash, content_type)| { + crate::application::ports::dedup_ports::DerivedBlobRef { + blob_hash, + content_type, + } + }) + } + + pub async fn store_derived_blob( + &self, + source_hash: &str, + kind: &str, + variant: &str, + content_type: &str, + bytes: Bytes, + ) -> Result { + let stored = self + .store_from_stream( + stream::once(async move { Ok::(bytes) }), + Some(content_type.to_string()), + ) + .await?; + let derived_hash = stored.hash().to_string(); + + let inserted = sqlx::query( + // The source must still EXIST, or this row can never be cleaned + // up. `purge_derived_blobs` runs from the source's reap, so a + // mapping written after that reap is unreachable forever: nothing + // will reap that hash a second time, and the orphaned row holds + // its derived blob's ref_count at 1, which GC is then correct to + // refuse. Permanent leak, three rows per image. + // + // It is not hypothetical. Background thumbnail generation is + // spawned and unawaited, so an upload deleted promptly — which a + // test suite does constantly, and users do occasionally — has its + // render finish AFTER the blob was reaped and then record a + // mapping to a corpse. + // + // Checking both tables because `source_hash` names a Blob: + // a manifest for CDC content, a bare blob row for legacy + // whole-file content. + // + // Zero rows here is indistinguishable from the ON CONFLICT case, + // and both want the same handling — release the reference the + // blob write just took — which the caller already does. + "INSERT INTO storage.content_derived_blobs + (source_hash, kind, variant, blob_hash, content_type) + SELECT $1, $2, $3, $4, $5 + WHERE EXISTS (SELECT 1 FROM storage.chunk_manifests WHERE file_hash = $1) + OR EXISTS (SELECT 1 FROM storage.blobs WHERE hash = $1) + ON CONFLICT (source_hash, kind, variant) DO NOTHING", + ) + .bind(source_hash) + .bind(kind) + .bind(variant) + .bind(&derived_hash) + .bind(content_type) + .execute(self.pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("Dedup", format!("record derived blob: {e}")))? + .rows_affected(); + + if inserted == 0 { + // Two causes, one correct response. + // + // Either someone else already mapped this variant (ON CONFLICT), + // or the source Blob no longer exists so the WHERE EXISTS above + // refused the row. Both leave our blob write with no mapping + // behind it, and in both cases keeping the reference would pin + // the blob forever — inflating ref_count on every re-render in + // the first case, stranding an unreachable blob in the second. + if let Err(e) = self.remove_reference(&derived_hash).await { + tracing::warn!( + target: "oxicloud::dedup", + error = %e, + "failed to release duplicate derived-blob reference for {}", + &derived_hash[..derived_hash.len().min(12)], + ); + } + } + + Ok(derived_hash) + } + + /// Look up a derived artifact by its source content. Read counterpart of + /// [`Self::store_derived_blob`]. + pub async fn find_derived_blob( + &self, + source_hash: &str, + kind: &str, + variant: &str, + ) -> Option { + match self.lookup_derived(source_hash, kind, variant).await { + crate::application::ports::dedup_ports::DerivedLookup::Found(r) => Some(r), + _ => None, + } + } + + /// Full three-way answer: no row, a negative verdict, or the blob. + /// + /// Callers deciding whether to spend a decode want the middle case, + /// which [`Self::find_derived_blob`] cannot express — it folds + /// "never attempted" and "attempted, not worth it" into the same + /// `None`, and a caller acting on that repeats the expensive work + /// forever. Use this wherever the derivation is costly; use + /// `find_derived_blob` when you only need the bytes. + /// + /// A query error reads as `Missing`, deliberately: a database blip + /// should cost a redundant render, never a wrong "not derivable" + /// that suppresses a derivation the content can support. + pub async fn lookup_derived( + &self, + source_hash: &str, + kind: &str, + variant: &str, + ) -> crate::application::ports::dedup_ports::DerivedLookup { + use crate::application::ports::dedup_ports::{DerivedBlobRef, DerivedLookup}; + + let row = sqlx::query_as::<_, (Option, Option)>( + "SELECT blob_hash, content_type FROM storage.content_derived_blobs + WHERE source_hash = $1 AND kind = $2 AND variant = $3", + ) + .bind(source_hash) + .bind(kind) + .bind(variant) + .fetch_optional(self.pool.as_ref()) + .await + .ok() + .flatten(); + + match row { + None => DerivedLookup::Missing, + // The CHECK constraint keeps blob_hash and content_type NULL + // together, so one NULL is the whole negative row. + Some((None, _)) | Some((_, None)) => DerivedLookup::NotDerivable, + Some((Some(blob_hash), Some(content_type))) => DerivedLookup::Found(DerivedBlobRef { + blob_hash, + content_type, + }), + } + } + + /// Record that this derivation is not worth attempting again. + /// + /// For outcomes that are deterministic in the source content — a + /// transcode that came out larger, a source that will not decode, a + /// source over the decode ceiling. **Never** for a timeout, a closed + /// semaphore, or an I/O error: those are properties of the moment, + /// and a row written for one marks good content underivable forever + /// with nothing to retry it. + /// + /// Takes no reference on any Blob — there is no derived Blob to hold + /// one. The row is dependent on its source and is reaped with it, + /// same as a positive row. + /// + /// Guarded by the same source-exists check as `store_derived_blob`: + /// a row whose source has already been reaped is a permanent leak of + /// a mapping nothing will ever clean up. + pub async fn store_derived_negative( + &self, + source_hash: &str, + kind: &str, + variant: &str, + ) -> Result<(), DomainError> { + sqlx::query( + "INSERT INTO storage.content_derived_blobs + (source_hash, kind, variant, blob_hash, content_type) + SELECT $1, $2, $3, NULL, NULL + WHERE EXISTS (SELECT 1 FROM storage.chunk_manifests WHERE file_hash = $1) + OR EXISTS (SELECT 1 FROM storage.blobs WHERE hash = $1) + ON CONFLICT (source_hash, kind, variant) DO NOTHING", + ) + .bind(source_hash) + .bind(kind) + .bind(variant) + .execute(self.pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Dedup", format!("store_derived_negative: {e}")) + })?; + Ok(()) + } + + /// The registry backing the reap predicate. + /// + /// Exposed so `blobs_consistency` recomputes refcounts from the *same* + /// source set GC reaps from. If the two ever diverged, the sweep would + /// bless counts the collector disagrees with — and the collector wins, + /// destructively. + pub fn reference_registry(&self) -> Arc { + self.reference_registry.clone() + } + /// Registers the blob lifecycle dispatcher (thumbnail cleanup, …). pub fn with_blob_lifecycle(mut self, lifecycle: Arc) -> Self { self.blob_lifecycle = Some(lifecycle); @@ -488,6 +993,89 @@ impl DedupService { } } + /// Everything that must happen when a blob is permanently reaped: + /// drop the artifacts derived FROM it, then notify the lifecycle hooks. + /// + /// Boxed because it is mutually recursive with `remove_reference`: + /// releasing a thumbnail's reference can reap the thumbnail's own blob, + /// which comes back through here. It terminates after one level — + /// nothing is derived from a thumbnail, so the inner purge finds no rows. + fn reap_blob<'a>( + &'a self, + hash: &'a str, + ) -> Pin + Send + 'a>> { + Box::pin(async move { + self.purge_derived_blobs(hash).await; + self.fire_blob_hooks(hash); + }) + } + + /// Delete every artifact derived from `source_hash` and release the + /// manifest references those rows held. + /// + /// The delete counterpart of [`Self::store_derived_blob`]. Without it a + /// thumbnail pins its own blob forever: the mapping row keeps + /// `chunk_manifests.ref_count` at 1 with no file behind it, so GC never + /// reclaims the bytes and a full delete leaves orphans on disk. + async fn purge_derived_blobs(&self, source_hash: &str) { + let derived: Vec<(String,)> = match sqlx::query_as( + "DELETE FROM storage.content_derived_blobs + WHERE source_hash = $1 + RETURNING blob_hash", + ) + .bind(source_hash) + .fetch_all(self.pool.as_ref()) + .await + { + Ok(rows) => rows, + Err(e) => { + tracing::warn!( + target: "oxicloud::dedup", + error = %e, + "failed to purge derived blobs for {}", + &source_hash[..source_hash.len().min(12)], + ); + return; + } + }; + + // Silent on success until now, which made three distinct outcomes + // indistinguishable from the outside: never called, called and found + // nothing, or found rows whose release then failed. Chasing an + // orphaned-derived-row leak cost several full suite runs for exactly + // that reason, so the call announces itself. + // + // `info` when it actually deleted something — that is rare (only when + // a source Blob dies) and it is the line that proves the reap path + // reached here. `debug` for the common no-op. + if derived.is_empty() { + tracing::debug!( + target: "oxicloud::dedup", + "purge_derived_blobs: no rows for {}", + &source_hash[..source_hash.len().min(12)], + ); + } else { + tracing::info!( + target: "oxicloud::dedup", + rows = derived.len(), + "purge_derived_blobs: releasing {} derived row(s) for {}", + derived.len(), + &source_hash[..source_hash.len().min(12)], + ); + } + + for (blob_hash,) in derived { + if let Err(e) = self.remove_reference(&blob_hash).await { + tracing::warn!( + target: "oxicloud::dedup", + error = %e, + "failed to release derived blob {}", + &blob_hash[..blob_hash.len().min(12)], + ); + } + } + } + fn fire_blob_hooks(&self, hash: &str) { if let Some(lc) = &self.blob_lifecycle { lc.on_blob_deleted(hash); @@ -510,12 +1098,16 @@ impl DedupService { .connect_lazy("postgres://invalid:5432/none") .unwrap(), ); + let stub_registry = Arc::new(Self::default_reference_registry(stub_pool.clone())); Self { backend: Arc::new(LocalBlobBackend::new(Path::new("/tmp/oxicloud_stub_blobs"))), pool: stub_pool.clone(), - maintenance_pool: stub_pool, + maintenance_pool: stub_pool.clone(), blob_lifecycle: None, manifest_cache: Self::build_manifest_cache(), + reference_registry: stub_registry.clone(), + manifest_reap_sql: manifest_reap_sql(&stub_registry), + blob_reap_sql: blob_reap_sql(&stub_registry), } } @@ -523,6 +1115,32 @@ impl DedupService { pub async fn initialize(&self) -> Result<(), DomainError> { self.backend.initialize().await?; + // The reap statement is assembled from the registered reference + // sources, so it is not greppable in the source tree. It DELETES + // manifests, so log it unconditionally at info rather than hiding it + // behind a filter an operator has to know to enable — if what GC + // considers "referenced" ever changes, that must be visible on the + // next boot without anyone going looking. + // + // Whitespace-collapsed to a single field so a multi-line query does + // not sprawl across the boot log; expand it with + // `sed 's/ AND / AND\n /g'` or just paste it into psql. + tracing::info!( + target: "oxicloud::dedup", + sources = ?self + .reference_registry + .sources() + .iter() + .map(|s| s.source_name()) + .collect::>(), + statement = %self + .manifest_reap_sql + .split_whitespace() + .collect::>() + .join(" "), + "🧹 manifest reap predicate registered" + ); + let blob_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM storage.blobs") .fetch_one(self.pool.as_ref()) .await @@ -601,7 +1219,15 @@ impl DedupService { /// chunks at `ref_count = 0` and is about to commit their manifest — cannot /// race the sweep. Must comfortably exceed the longest plausible gap /// between registering a chunk and referencing it (any in-flight upload). - const GC_ORPHAN_GRACE_SECS: i64 = 60 * 60; // 1 hour + /// + /// `pub` because sibling consistency tenants derive their own grace + /// windows from this value — notably `blobs_consistency`'s + /// `blob_orphan_stalled` check, which flags rows that have been sitting + /// past `GC_ORPHAN_GRACE_SECS × 24` (a healthy sweep would never trip + /// that). Keeping the two grace values coupled at the constant, rather + /// than at two hand-tuned magic numbers, means tuning this one auto- + /// scales the stall threshold too. + pub const GC_ORPHAN_GRACE_SECS: i64 = 60 * 60; // 1 hour /// Store content with CDC deduplication, straight from a byte stream — /// the single write path for every upload surface (REST multipart, @@ -1722,7 +2348,7 @@ impl DedupService { self.manifest_cache.invalidate(file_hash).await; // File content is gone — drop its blob-keyed thumbnails now. - self.fire_blob_hooks(file_hash); + self.reap_blob(file_hash).await; tracing::info!( "MANIFEST DELETED: {} ({} chunks dereferenced; orphans reclaimed by GC)", @@ -1801,7 +2427,7 @@ impl DedupService { } // Bug 3 fix: notify hooks — e.g. thumbnail cleanup keyed by hash - self.fire_blob_hooks(hash); + self.reap_blob(hash).await; tracing::info!("BLOB DELETED: {} (no more references)", &hash[..12]); Ok(true) @@ -1838,66 +2464,48 @@ impl DedupService { pub async fn cleanup_if_orphaned(&self, hash: &str) { let short = &hash[..hash.len().min(12)]; - // ── CDC manifest path (must run FIRST) ─────────────────── - // For single-chunk CDC files file_hash == chunk_hash, so the PG - // trigger on storage.files already decremented storage.blobs.ref_count - // when this function is called. try_dedup_hit increments - // chunk_manifests.ref_count but NOT storage.blobs.ref_count, so - // blobs.ref_count can reach 0 while the manifest still has ref_count > 1 - // (other files sharing the same blob). Checking the manifest first - // prevents premature blob + manifest deletion. - let manifest = sqlx::query_as::<_, (i32, Vec)>( - "SELECT ref_count, chunk_hashes \ - FROM storage.chunk_manifests WHERE file_hash = $1", - ) - .bind(hash) - .fetch_optional(self.pool.as_ref()) - .await - .unwrap_or(None); - - if let Some((ref_count, chunk_hashes)) = manifest { - if ref_count <= 1 { - // Last reference — remove manifest and all its chunks. - if let Err(e) = self - .remove_manifest_reference(hash, ref_count, &chunk_hashes) - .await - { - tracing::warn!("cleanup_if_orphaned: manifest cleanup failed for {short}: {e}"); - } - } else { - // Other files still share this blob: just decrement the manifest - // counter and undo the PG trigger's premature chunk ref_count - // decrement (blobs.ref_count is chunk-level; the manifest is the - // authoritative file-level counter). - sqlx::query( - "UPDATE storage.chunk_manifests \ - SET ref_count = ref_count - 1 WHERE file_hash = $1", - ) - .bind(hash) - .execute(self.pool.as_ref()) - .await - .ok(); - // Undo the PG trigger's decrement of storage.blobs.ref_count. - // The trigger fired with blob_hash = file_hash, so only the row - // WHERE hash = file_hash is affected. For single-chunk files - // file_hash == chunk_hash and that row exists; for multi-chunk - // files file_hash is not in storage.blobs, making this a no-op. - sqlx::query("UPDATE storage.blobs SET ref_count = ref_count + 1 WHERE hash = $1") - .bind(hash) - .execute(self.pool.as_ref()) - .await - .ok(); - tracing::debug!( - "cleanup_if_orphaned: manifest {short} ref_count {ref_count}→{}", - ref_count - 1 - ); - } - return; - } - - // ── Legacy blob path (no manifest) ─────────────────────── + // 2026-08-23 refactor: this function used to compensate for the + // OLD PG trigger `trg_files_decrement_blob_ref` unconditionally + // decrementing `storage.blobs.ref_count`, which was wrong for + // CDC files (their `blob_hash` names a `chunk_manifests.file_hash`, + // not a chunk-in-a-manifest). The compensation branches would: + // * Decrement `chunk_manifests.ref_count` a SECOND time (the + // trigger having wrongly touched blobs, not the manifest); + // * Undo the trigger's blob decrement (rc > 1 branch); + // * Call `remove_manifest_reference` (rc <= 1 branch), which + // deletes manifest + dereferences chunks — again duplicating + // work the trigger should own. + // + // Migration `20261017000000_file_delete_trigger_manifest_aware.sql` + // rewrote the trigger to be manifest-aware, so it now correctly + // decrements EITHER the manifest OR the blob depending on which + // one the hash names, walks chunks on last-ref manifest delete, + // and leaves the counters in a consistent state without any + // compensation call. Running the old compensation ON TOP of the + // new trigger causes double-decrement / double-delete and is + // exactly what broke `dedup_blob_cleanup.hurl` step 7 + // (`ref_count == 1` observed 0 after purging one of two dedup + // uploads). + // + // What remains here: **physical cleanup only**. If the trigger + // brought a LEGACY whole-file blob to ref_count = 0 and no + // manifest still references it (either directly via file_hash or + // indirectly as a chunk in another manifest's chunk_hashes[]), + // reap the DB row and the backend file eagerly. For CDC chunks + // whose ref_count reached 0 via the trigger's last-ref manifest + // path, `dedup_gc` handles physical reap with a grace window + // against re-upload races. + // + // Callers can keep invoking `cleanup_if_orphaned` unconditionally + // — for CDC paths it's a cheap no-op (manifest still exists OR + // the hash never had a blob row), for legacy paths it reaps. let deleted_blob = sqlx::query_scalar::<_, String>( - "DELETE FROM storage.blobs WHERE hash = $1 AND ref_count <= 0 RETURNING hash", + "DELETE FROM storage.blobs \ + WHERE hash = $1 \ + AND ref_count <= 0 \ + AND NOT EXISTS (SELECT 1 FROM storage.chunk_manifests \ + WHERE $1 = ANY(chunk_hashes)) \ + RETURNING hash", ) .bind(hash) .fetch_optional(self.pool.as_ref()) @@ -1908,8 +2516,8 @@ impl DedupService { if let Err(e) = self.backend.delete_blob(hash).await { tracing::warn!("cleanup_if_orphaned: disk delete failed for {short}: {e}"); } - self.fire_blob_hooks(hash); - tracing::info!("cleanup_if_orphaned: removed orphaned blob {short}"); + self.reap_blob(hash).await; + tracing::info!("cleanup_if_orphaned: removed orphaned legacy blob {short}"); } } @@ -2555,13 +3163,18 @@ impl DedupService { let mut total_bytes = 0u64; // ── Phase 1: GC orphaned manifests ─────────────────────── - // A manifest is collectible when: - // • ref_count has been decremented to 0 by cleanup_if_orphaned - // on the single-file-delete service path, OR - // • no `storage.files.blob_hash` references its file_hash - // (covers bulk-delete paths: user cascade, empty_trash — - // where the PG trigger only touches storage.blobs and the - // per-file cleanup_if_orphaned call is skipped). + // A manifest is collectible when NO registered reference source + // references its file_hash. That single condition covers both + // delete paths: the single-file service path removes the + // storage.files row, and so do the bulk paths (user cascade, + // empty_trash) — whichever decrements ref_count along the way is + // irrelevant here. + // + // ref_count is deliberately NOT part of this. It used to be, as + // `ref_count <= 0 OR `, which meant a counter that + // under-reported deleted live content without ever consulting the + // registry that knew better. See `manifest_reap_sql` for the full + // reasoning and for what moved to the refcount recompute instead. loop { // Keep the historically cheap DELETE-only shape for the dominant // no-work sweep. Embedding it in the delete/aggregate/update CTE @@ -2570,23 +3183,14 @@ impl DedupService { // update. From two onward, aggregate in-process and issue one UPDATE: // the measured crossover is already positive at two, while 500 and // 1,000 manifests improve by 60.03x and 51.16x respectively. - let batch: Vec<(String, Vec, i64)> = sqlx::query_as( - "DELETE FROM storage.chunk_manifests - WHERE ctid = ANY( - SELECT ctid FROM storage.chunk_manifests m - WHERE m.ref_count <= 0 - OR NOT EXISTS ( - SELECT 1 FROM storage.files f - WHERE f.blob_hash = m.file_hash - ) - LIMIT $1 - ) - RETURNING file_hash, chunk_hashes, total_size", - ) - .bind(BATCH_SIZE) - .fetch_all(self.maintenance_pool.as_ref()) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("GC manifests: {e}")))?; + // Assembled once at construction (see `manifest_reap_sql`), not + // per sweep: no string work in the hot path, a stable statement for + // prepared-statement caching, and a byte-for-byte golden test. + let batch: Vec<(String, Vec, i64)> = sqlx::query_as(&self.manifest_reap_sql) + .bind(BATCH_SIZE) + .fetch_all(self.maintenance_pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("Dedup", format!("GC manifests: {e}")))?; if batch.is_empty() { break; @@ -2599,6 +3203,28 @@ impl DedupService { // and accounting remain below and run only after refcounts succeed. for (file_hash, _, _) in &batch { self.manifest_cache.invalidate(file_hash).await; + + // Drop everything derived FROM this Blob, exactly as + // `reap_blob` does for the single-blob path. + // + // Without this, bulk manifest reaping orphans the rows: the + // reap predicate protects a manifest that IS a derived + // artifact (`content_derived_blobs.blob_hash`), but + // deliberately not one that is the SOURCE of them — counting + // `source_hash` as a reference would pin every original for + // as long as a thumbnail existed. So the source is reaped + // correctly, and the purge has to follow it. + // + // It did not, and the leak is permanent rather than cosmetic: + // the orphaned row holds `chunk_manifests.ref_count` at 1 on + // the thumbnail's own blob, so GC is thereafter *correct* to + // refuse it and those bytes are never reclaimed. Every + // deleted image left three of them behind — one per size. + // + // Found by storage_cleanup_check.sh: three leftover blobs, + // all `derived=1`, all naming one `src` whose manifest, blob + // row and files were already gone. + self.purge_derived_blobs(file_hash).await; } if batch.len() == 1 { @@ -2658,7 +3284,7 @@ impl DedupService { // chunk-keyed hook never finds them. Symptom: orphan webp // under `.thumbnails/{icon,preview,large}/.webp` // after a user-cascade-delete of a video upload. - self.fire_blob_hooks(file_hash); + self.reap_blob(file_hash).await; total_bytes += *size as u64; tracing::debug!( @@ -2679,41 +3305,29 @@ impl DedupService { // NULL orphaned_at — a pre-migration row or a path that never // stamped it; those are safe to take immediately), AND // • no manifest still lists it as a chunk, AND - // • no file still points at it directly (legacy whole-file blob). + // • no file still points at it directly (legacy whole-file blob), + // AND + // • no registered reference source claims it at the chunk level. // - // The two NOT EXISTS guards mirror Phase 1's file cross-check: a stale - // ref_count = 0 on still-referenced content can then only delay - // collection, never delete live bytes. The grace window keeps a + // The NOT EXISTS guards mean a stale ref_count = 0 on still-referenced + // content can only delay collection, never delete live bytes — unlike + // Phase 1 before `manifest_reap_sql` dropped its ref_count arm, this + // phase always had that property. The registry conjunct is additive + // (see `blob_reap_sql`): it cannot reap anything the hardcoded guards + // would have spared, it just stops a future chunk-level source from + // being missed. The grace window keeps a // concurrent uploader that is about to pin a just-orphaned chunk from // racing the row-delete → file-unlink gap (see GC_ORPHAN_GRACE_SECS). // The ctid snapshot already protects against a pin that commits DURING // the DELETE (the pin rewrites the row's ctid, so it drops out of the // set); grace covers the remaining post-commit unlink window. loop { - let batch: Vec<(String, i64)> = sqlx::query_as( - "DELETE FROM storage.blobs - WHERE ctid = ANY( - SELECT b.ctid FROM storage.blobs b - WHERE b.ref_count <= 0 - AND (b.orphaned_at IS NULL - OR b.orphaned_at < now() - ($2::int * interval '1 second')) - AND NOT EXISTS ( - SELECT 1 FROM storage.chunk_manifests m - WHERE m.chunk_hashes @> ARRAY[b.hash::text] - ) - AND NOT EXISTS ( - SELECT 1 FROM storage.files f - WHERE f.blob_hash = b.hash - ) - LIMIT $1 - ) - RETURNING hash, size", - ) - .bind(BATCH_SIZE) - .bind(grace_secs as i32) - .fetch_all(self.maintenance_pool.as_ref()) - .await - .map_err(|e| DomainError::internal_error("Dedup", format!("GC blobs: {e}")))?; + let batch: Vec<(String, i64)> = sqlx::query_as(&self.blob_reap_sql) + .bind(BATCH_SIZE) + .bind(grace_secs as i32) + .fetch_all(self.maintenance_pool.as_ref()) + .await + .map_err(|e| DomainError::internal_error("Dedup", format!("GC blobs: {e}")))?; if batch.is_empty() { break; @@ -2741,7 +3355,7 @@ impl DedupService { .await; for (hash, size) in &deleted { - self.fire_blob_hooks(hash); + self.reap_blob(hash).await; total_bytes += *size as u64; } total_deleted += n as u64; @@ -3126,6 +3740,15 @@ impl DedupPort for DedupService { self.blob_exists(hash).await } + async fn find_derived_blob( + &self, + source_hash: &str, + kind: &str, + variant: &str, + ) -> Option { + self.find_derived_blob(source_hash, kind, variant).await + } + async fn get_blob_metadata(&self, hash: &str) -> Option { self.get_blob_metadata(hash).await } @@ -3220,6 +3843,20 @@ impl crate::infrastructure::scheduler::JobHandler for DedupService { DEDUP_GC_JOB_NAME } + fn description(&self) -> &'static str { + "Reclaims blobs and chunk manifests that no file, thumbnail or \ + preview references any more, once they are past the orphan grace \ + window. Trash cleanup already runs this as its tail step; \ + triggering it here is for reclaiming immediately rather than at \ + the next tick. Add ?force=true to skip the grace window." + } + + /// Deletes bytes. `force` is its accelerator, not a repair flag — + /// there is nothing this job reports without also acting on it. + fn mutates(&self) -> crate::infrastructure::scheduler::Mutates { + crate::infrastructure::scheduler::Mutates::Always + } + /// Runs one `garbage_collect` sweep — the same reclamation that /// `TrashCleanupService` invokes inline as its tail step, exposed /// through the scheduler so operators can trigger it uniformly via @@ -3267,6 +3904,144 @@ impl crate::infrastructure::scheduler::JobHandler for DedupService { #[cfg(test)] mod tests { use super::*; + + /// Golden test for the statement `garbage_collect` runs against production + /// data. It is assembled from the registered reference sources rather than + /// written as a literal, so this pins the whole thing byte-for-byte — the + /// point being that a reviewer reads the SQL *here* instead of mentally + /// evaluating the registry. + /// + /// If this fails after adding a source, read the diff carefully: the new + /// branch must appear inside the `NOT (...)` group, ORed with the others. + /// A branch landing outside that group inverts the predicate for every + /// other source and reaps live manifests. + /// + /// **`ref_count` must not reappear in this statement.** It used to be + /// there as `ref_count <= 0 OR NOT (…)`, which let a counter that + /// under-reported delete content the registry still knew was referenced. + /// If a future change reintroduces it, this test fails, and that failure + /// is the point — see `manifest_reap_sql` and + /// `gc_reference_authority_integration_tests`. + #[tokio::test] + async fn manifest_reap_statement_is_stable() { + let sql = DedupService::new_stub().manifest_reap_sql; + let expected = r#"DELETE FROM storage.chunk_manifests + WHERE ctid = ANY( + SELECT ctid + FROM storage.chunk_manifests m + WHERE NOT (EXISTS (SELECT 1 FROM storage.files cnt_f WHERE cnt_f.blob_hash = m.file_hash) + OR EXISTS (SELECT 1 FROM storage.content_derived_blobs cnt_d WHERE cnt_d.blob_hash = m.file_hash) + OR EXISTS (SELECT 1 FROM storage.file_attached_blobs cnt_a WHERE cnt_a.blob_hash = m.file_hash)) + LIMIT $1 + ) + RETURNING file_hash, chunk_hashes, total_size"#; + assert_eq!(sql, expected, "reap statement changed:\n{sql}"); + assert!( + !sql.contains("ref_count"), + "ref_count is back in the reap predicate — the counter must not be \ + able to delete data on its own" + ); + } + + /// The reap predicate must never match a manifest that some source still + /// references. With an empty registry `NOT (...)` would have no operands, + /// so the builder refuses rather than emitting a statement that deletes + /// every manifest in the database. + #[test] + #[should_panic(expected = "no manifest-level blob reference source")] + fn empty_registry_refuses_to_build_reap_statement() { + let _ = manifest_reap_sql(&BlobReferenceRegistry::new()); + } + + #[test] + #[should_panic(expected = "no chunk-level blob reference source")] + fn empty_registry_refuses_to_build_blob_reap_statement() { + let _ = blob_reap_sql(&BlobReferenceRegistry::new()); + } + + /// Golden test for GC phase 2, same purpose as the manifest one. + /// + /// Note what this pins that the manifest statement does not: the two + /// hardcoded `NOT EXISTS` guards **and** the registry predicate, ANDed. + /// The registry fragment is not a replacement here — see `blob_reap_sql` + /// for why substituting it would reap a legacy blob row mid-rechunk. + #[tokio::test] + async fn blob_reap_statement_is_stable() { + let sql = DedupService::new_stub().blob_reap_sql; + let expected = r#"DELETE FROM storage.blobs + WHERE ctid = ANY( + SELECT b.ctid FROM storage.blobs b + WHERE b.ref_count <= 0 + AND (b.orphaned_at IS NULL + OR b.orphaned_at < now() - ($2::int * interval '1 second')) + AND NOT EXISTS ( + SELECT 1 FROM storage.chunk_manifests m + WHERE m.chunk_hashes @> ARRAY[b.hash::text] + ) + AND NOT EXISTS ( + SELECT 1 FROM storage.files f + WHERE f.blob_hash = b.hash + ) + AND NOT (EXISTS (SELECT 1 FROM storage.files cnt_f WHERE cnt_f.blob_hash = b.hash AND NOT EXISTS (SELECT 1 FROM storage.chunk_manifests cnt_m WHERE cnt_m.file_hash = cnt_f.blob_hash)) + OR EXISTS (SELECT 1 FROM storage.chunk_manifests cnt_m WHERE b.hash = ANY(cnt_m.chunk_hashes))) + LIMIT $1 + ) + RETURNING hash, size"#; + assert_eq!(sql, expected, "blob reap statement changed:\n{sql}"); + } + + /// The reason phase 2 became registry-driven at all. + /// + /// Today no source contributes at [`RefLevel::Chunk`] beyond files and + /// manifests, so the registry conjunct is operationally redundant and a + /// golden test alone would not notice if it stopped being wired up. This + /// registers a synthetic chunk-level source and asserts its fragment + /// reaches the statement — which is what stops a future + /// `content_derived_blobs`-style table from being silently missed the way + /// Phase 1's hardcoded cross-check missed them. + #[tokio::test] + async fn a_new_chunk_level_source_reaches_the_blob_reap_statement() { + use crate::application::ports::blob_reference_ports::BlobReferenceSource; + + struct FakeChunkSource; + + #[async_trait::async_trait] + impl BlobReferenceSource for FakeChunkSource { + fn source_name(&self) -> &'static str { + "fake_chunk_source" + } + fn ref_count_sql(&self, level: RefLevel, outer: &str) -> Option { + self.ref_exists_sql(level, outer) + } + fn ref_exists_sql(&self, level: RefLevel, outer: &str) -> Option { + match level { + RefLevel::Chunk => Some(format!( + "EXISTS (SELECT 1 FROM storage.zzz_fake WHERE blob_hash = {outer})" + )), + RefLevel::Manifest => None, + } + } + async fn count_references(&self, _hash: &str) -> Result { + Ok(0) + } + async fn list_referenced_blobs( + &self, + _cursor: Option>, + _limit: usize, + ) -> Result<(Vec, Option>), DomainError> { + Ok((Vec::new(), None)) + } + } + + let mut registry = BlobReferenceRegistry::new(); + registry.register(Arc::new(FakeChunkSource)); + let sql = blob_reap_sql(®istry); + + assert!( + sql.contains("storage.zzz_fake"), + "a chunk-level source must reach the phase-2 reap guard:\n{sql}" + ); + } use std::collections::HashSet; use tempfile::NamedTempFile; @@ -4100,6 +4875,30 @@ mod rechunk_integration_tests { } } +/// Serializes every integration test that runs a **global** GC sweep. +/// +/// GC sweeps the shared integration database, while each test intentionally +/// owns a different `TempDir`-backed blob store. Two sweep tests running +/// concurrently can therefore delete test A's row through test B's backend, +/// leaving A's physical blob behind and failing an assertion that has nothing +/// to do with the code under test. Production has one shared backend for the +/// swept database; serializing only these tests models that invariant. +/// +/// **Any new test that calls `garbage_collect*` must take this guard**, +/// wherever it lives in this file. It sat inside +/// `delta_upload_integration_tests` until `gc_reference_authority_integration_tests` +/// was added without it and broke +/// `garbage_collect_honours_grace_window_and_references` — a failure that +/// appeared only in the full suite and pointed at the wrong test. Hoisted to +/// module scope so the next suite finds it. +/// +/// `allow(dead_code)`: gated on a cfg flag rather than on `test`, so a plain +/// build with `--cfg integration_tests` compiles it while `#[tokio::test]` +/// drops every caller. +#[cfg(integration_tests)] +#[allow(dead_code)] +static GC_TEST_SERIALIZER: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + // ───────────────────────────────────────────────────────────────────────────── // Integration tests for the delta-upload primitives — the entitlement and // verification rules the chunk-negotiation protocol stands on. Same gating @@ -4116,14 +4915,6 @@ mod delta_upload_integration_tests { use tempfile::TempDir; use uuid::Uuid; - // GC sweeps the shared integration database globally, while every test - // intentionally owns a different TempDir-backed blob store. Running two - // sweep tests concurrently can therefore delete test A's row through test - // B's backend, leaving A's physical blob behind. Production has one shared - // backend for the swept database; serialize only these global-sweep tests - // so the integration topology models that invariant. - static GC_TEST_SERIALIZER: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); - async fn test_pool() -> Arc { let pool = PgPoolOptions::new() .max_connections(4) @@ -4740,3 +5531,326 @@ mod delta_upload_integration_tests { cleanup(&pool, &file_hash, file_id, &[]).await; } } + +// ───────────────────────────────────────────────────────────────────────────── +// Who decides a manifest is dead: the counter, or the reference registry? +// +// **The registry, and only the registry.** `manifest_reap_sql` asks +// `WHERE ` and does not mention +// `ref_count` at all. +// +// It used to read `ref_count <= 0 OR `. Each arm covered a +// real deletion path — the single-file path decrements the counter via +// `cleanup_if_orphaned`, bulk paths (user cascade, empty_trash) only fire the +// `storage.blobs` trigger — so the disjunction looked like belt and braces. +// It was the opposite: with OR, either signal alone deletes, so a counter +// that under-reported made live content collectible and the registry that +// knew better was never consulted. +// +// Not hypothetical. `storage.copy_folder_tree` used to take references with +// `UPDATE storage.blobs … WHERE hash = blob_hash`, which matches nothing for +// a CDC file — whose `blob_hash` names a manifest, not a chunk — so it took +// no reference at all. Copy a folder, delete the original, and the copy's +// bytes were reaped. Both copy paths now go through +// `storage.add_blob_references`, but that fix relied on getting the counter +// right, and there are two implementations of the reference contract +// (`storage.add_blob_references` in SQL, `DedupService::add_reference` in +// Rust) that must agree forever. Removing the counter's authority is what +// makes a future disagreement a leak rather than data loss. +// +// The two tests pin both directions, and they are only meaningful together: +// +// * `gc_spares_a_manifest_with_a_live_referrer` — a wrong-LOW counter must +// not delete. This is the fix. +// * `gc_reaps_an_unreferenced_manifest_despite_a_high_refcount` — a +// wrong-HIGH counter must not veto. This is the coverage the removed arm +// used to provide, and dropping it must not have traded one failure for +// the other. +// +// See `docs/plan/derived-blobs.md`. Gated on `--cfg integration_tests` like +// the other PG suites. +// ───────────────────────────────────────────────────────────────────────────── +// `allow(dead_code)`: the module is gated on a cfg flag, not on `test`, so a +// plain `cargo build --cfg integration_tests` compiles the helpers while +// `#[tokio::test]` drops their only callers. Same reason the rechunk suite +// above carries it. +#[cfg(integration_tests)] +#[allow(dead_code)] +mod gc_reference_authority_integration_tests { + use super::*; + use crate::infrastructure::services::local_blob_backend::LocalBlobBackend; + use crate::integration_test_support::{ensure_clean_test_db, test_db_url}; + use sqlx::Row; + use sqlx::postgres::PgPoolOptions; + use tempfile::TempDir; + use uuid::Uuid; + + async fn test_pool() -> Arc { + let pool = PgPoolOptions::new() + .max_connections(4) + .connect(&test_db_url()) + .await + .expect("connect to test DB — run tests/common/spawn-db.sh first"); + ensure_clean_test_db(&pool).await; + Arc::new(pool) + } + + async fn seed_user(pool: &PgPool) -> Uuid { + sqlx::query("SELECT d.id AS drive_id FROM storage.drives d WHERE d.default_for_user IS NOT NULL LIMIT 1") + .fetch_one(pool) + .await + .map(|r| r.get::("drive_id")) + .expect("storage.drives must be seeded (init-test-schema.sh)") + } + + async fn local_svc(pool: &Arc, dir: &TempDir) -> DedupService { + let backend = Arc::new(LocalBlobBackend::new(&dir.path().join("blobs"))); + backend.initialize().await.expect("init backend"); + DedupService::new(backend, pool.clone(), pool.clone()) + } + + /// Unique, poorly-compressible content of `len` bytes. The random tail + /// keeps every invocation's hash distinct, so rows left behind by a + /// panicking run can never collide with the current one. + fn content(len: usize) -> Vec { + let mut data: Vec = (0..len) + .map(|i| ((i % 251) as u8).wrapping_add((i / 7919) as u8)) + .collect(); + data.extend_from_slice(Uuid::new_v4().as_bytes()); + data + } + + /// A stored CDC blob plus a live `storage.files` row referencing it. + /// + /// The file row is inserted BEFORE the store, deliberately: phase 1 of + /// `garbage_collect` reaps manifests no source references, so with the + /// opposite order a concurrent GC from another test could reap ours in + /// the window between the two statements. BLAKE3 is deterministic, so + /// the hash is known in advance and the order costs nothing. + /// + /// Returns `(file_hash, chunk_hashes, file_id)`. + async fn seed_referenced_cdc_blob( + svc: &DedupService, + pool: &PgPool, + drive_id: Uuid, + data: &[u8], + label: &str, + ) -> (String, Vec, Uuid) { + let file_hash = blake3::hash(data).to_hex().to_string(); + + let file_id: Uuid = sqlx::query_scalar( + "INSERT INTO storage.files (name, drive_id, blob_hash, size) + VALUES ($1, $2, $3, $4) RETURNING id", + ) + .bind(format!( + "rust-test-gcauth-{label}-{}", + &Uuid::new_v4().to_string()[..8] + )) + .bind(drive_id) + .bind(&file_hash) + .bind(data.len() as i64) + .fetch_one(pool) + .await + .expect("file row"); + + let source = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::copy_from_slice(data))]); + let stored = svc + .store_from_stream(source, Some("application/octet-stream".into())) + .await + .expect("store"); + assert_eq!( + stored.hash(), + file_hash, + "pre-computed BLAKE3 must match CDC-store output" + ); + + let chunks: Vec = sqlx::query_scalar( + "SELECT UNNEST(chunk_hashes) FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(&file_hash) + .fetch_all(pool) + .await + .expect("chunks"); + + // Fixture premise. A single-chunk blob has `file_hash == chunk_hash` + // (both BLAKE3 over the same bytes), which is the aliasing case the + // reference contract carries a `NOT EXISTS` guard for. This suite is + // about the multi-chunk shape — the one the copy bug broke, where + // `blob_hash` names a manifest that `storage.blobs` has no row for — + // so assert we actually got it rather than silently testing the easy + // case if CDC parameters change. + assert!( + chunks.len() > 1, + "fixture must be multi-chunk to exercise the manifest level, got {} \ + chunk(s) for {} bytes (CDC_AVG_CHUNK = {CDC_AVG_CHUNK})", + chunks.len(), + data.len() + ); + + (file_hash, chunks, file_id) + } + + async fn manifest_exists(pool: &PgPool, file_hash: &str) -> bool { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM storage.chunk_manifests WHERE file_hash = $1", + ) + .bind(file_hash) + .fetch_one(pool) + .await + .expect("count manifests") + > 0 + } + + /// Simulate a reference that was never taken: the file row is live, the + /// counter says nothing needs the content. Exactly the state the + /// `copy_folder_tree` bug produced, and the state any future divergence + /// between the SQL and Rust reference contracts would produce. + async fn force_zero_manifest_refcount(pool: &PgPool, file_hash: &str) { + let updated = + sqlx::query("UPDATE storage.chunk_manifests SET ref_count = 0 WHERE file_hash = $1") + .bind(file_hash) + .execute(pool) + .await + .expect("zero the manifest refcount") + .rows_affected(); + assert_eq!(updated, 1, "expected exactly one manifest for {file_hash}"); + } + + async fn cleanup(pool: &PgPool, file_hash: &str, file_id: Uuid, chunks: &[String]) { + let _ = sqlx::query("DELETE FROM storage.files WHERE id = $1") + .bind(file_id) + .execute(pool) + .await; + let _ = sqlx::query( + "DELETE FROM storage.files + WHERE blob_hash = $1 AND name LIKE 'rust-test-gcauth-%'", + ) + .bind(file_hash) + .execute(pool) + .await; + let _ = sqlx::query("DELETE FROM storage.chunk_manifests WHERE file_hash = $1") + .bind(file_hash) + .execute(pool) + .await; + let mut to_drop = chunks.to_vec(); + to_drop.push(file_hash.to_string()); + let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = ANY($1)") + .bind(&to_drop) + .execute(pool) + .await; + } + + /// The coverage that dropping the `ref_count` arm had to preserve. + /// + /// Bulk-delete paths (user cascade, `empty_trash`) remove + /// `storage.files` rows via a trigger that only touches `storage.blobs`, + /// so the manifest's counter is left **stuck high** with no referrers. + /// Under the old `OR` predicate the registry arm collected those. Now + /// that the registry is the sole authority it still does — a high counter + /// no longer keeps dead content alive, just as a zero one no longer kills + /// live content. + /// + /// This is the direction the counter can still be wrong in, and it is the + /// benign one: a leak, detected by the refcount recompute, not data loss. + #[tokio::test] + async fn gc_reaps_an_unreferenced_manifest_despite_a_high_refcount() { + let _gc_test_guard = GC_TEST_SERIALIZER.lock().await; + let pool = test_pool().await; + let drive_id = seed_user(&pool).await; + let dir = TempDir::new().expect("tempdir"); + let svc = local_svc(&pool, &dir).await; + + let data = content(2 * 1024 * 1024); + let (file_hash, chunks, file_id) = + seed_referenced_cdc_blob(&svc, &pool, drive_id, &data, "stuckhigh").await; + + // Simulate the bulk path: referrer gone, counter untouched. + sqlx::query("DELETE FROM storage.files WHERE id = $1") + .bind(file_id) + .execute(pool.as_ref()) + .await + .expect("drop the referrer"); + let bumped = + sqlx::query("UPDATE storage.chunk_manifests SET ref_count = 7 WHERE file_hash = $1") + .bind(&file_hash) + .execute(pool.as_ref()) + .await + .expect("inflate the refcount") + .rows_affected(); + assert_eq!(bumped, 1, "expected exactly one manifest for {file_hash}"); + + // Plain GC, NOT `garbage_collect_force`. Phase 1 has no time filter — + // the manifest predicate is purely "is it referenced" — so the grace + // window is irrelevant to what these tests assert. Forcing it would + // bypass the CHUNK-level grace for the whole shared test database and + // reap sibling tests' just-uploaded orphans; that is exactly how this + // suite first broke `claim_and_pin_respect_ownership_and_orphans`. + svc.garbage_collect().await.expect("gc"); + + let survived = manifest_exists(&pool, &file_hash).await; + cleanup(&pool, &file_hash, file_id, &chunks).await; + + assert!( + !survived, + "GC left a manifest nothing references, because its ref_count was \ + above zero. Removing the `ref_count <= 0` arm must not have made \ + the counter able to VETO collection either — the registry is the \ + authority in both directions." + ); + } + + /// **The contract.** A manifest with a live `storage.files` referrer + /// survives GC no matter what its counter says. + /// + /// This failed until `manifest_reap_sql` dropped its `ref_count <= 0` + /// arm. The counter was a second, independent licence to delete, so a + /// reference that was never taken — the `copy_folder_tree` bug — destroyed + /// the copy's content rather than merely mis-reporting a number. + #[tokio::test] + async fn gc_spares_a_manifest_with_a_live_referrer() { + let _gc_test_guard = GC_TEST_SERIALIZER.lock().await; + let pool = test_pool().await; + let drive_id = seed_user(&pool).await; + let dir = TempDir::new().expect("tempdir"); + let svc = local_svc(&pool, &dir).await; + + let data = content(2 * 1024 * 1024); + let (file_hash, chunks, file_id) = + seed_referenced_cdc_blob(&svc, &pool, drive_id, &data, "spare").await; + + force_zero_manifest_refcount(&pool, &file_hash).await; + + // The file row is still there — this is the whole premise, so assert + // it rather than trusting that nothing else reaped it concurrently. + let referrers: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM storage.files WHERE id = $1") + .bind(file_id) + .fetch_one(pool.as_ref()) + .await + .expect("count referrers"); + assert_eq!( + referrers, 1, + "fixture file row must still reference the blob" + ); + + // Plain GC — see the sibling test for why `force` is wrong here. + svc.garbage_collect().await.expect("gc"); + + let survived = manifest_exists(&pool, &file_hash).await; + let readable = svc.read_blob_stream(&file_hash).await.is_ok(); + cleanup(&pool, &file_hash, file_id, &chunks).await; + + assert!( + survived, + "GC reaped a manifest that storage.files still references. \ + ref_count was 0 and something let that alone decide — check \ + whether `manifest_reap_sql` has regained a `ref_count` clause. \ + FilesReferenceSource is registered and knows the row is live; it \ + must be the only authority on collectibility." + ); + assert!( + readable, + "manifest survived but its content is unreadable — chunk-level \ + reclamation followed the same zero counter" + ); + } +} diff --git a/src/infrastructure/services/drives_consistency_service.rs b/src/infrastructure/services/drives_consistency_service.rs index ea9da111..ed6b6dde 100644 --- a/src/infrastructure/services/drives_consistency_service.rs +++ b/src/infrastructure/services/drives_consistency_service.rs @@ -72,6 +72,13 @@ impl RecoverableJobHandler for DrivesConsistencyCheck { DRIVES_CONSISTENCY_JOB_NAME } + fn description(&self) -> &'static str { + "Compares each drive's cached used_bytes against the actual sum of \ + its file sizes and reports the drift. Read-only — usage_reconcile \ + is what corrects the counter; this surfaces WHEN it drifts so the \ + cause can be traced (missed delta, silent failure, race)." + } + /// Definitive count — one row per drive, table is tiny (dozens per /// install), COUNT(*) is trivially fast. Enables progress bar on /// the admin UI. diff --git a/src/infrastructure/services/entry_backend.rs b/src/infrastructure/services/entry_backend.rs index d9f02afc..005fb68b 100644 --- a/src/infrastructure/services/entry_backend.rs +++ b/src/infrastructure/services/entry_backend.rs @@ -168,7 +168,7 @@ pub async fn resolve_active_entry<'a>( "auth.admin_settings.storage.active_backend_name = `{name}`, but no entry \ with that name is declared in OXICLOUD_STORAGE_ENTRIES. Available: [{available}]. \ Either add `{name}` back to your .env, or repair the DB pointer with:\n \ - oxicloud --select-storage " + oxicloud storage select " )) } }, diff --git a/src/infrastructure/services/files_consistency_service.rs b/src/infrastructure/services/files_consistency_service.rs index aec953b9..59cdf61d 100644 --- a/src/infrastructure/services/files_consistency_service.rs +++ b/src/infrastructure/services/files_consistency_service.rs @@ -148,6 +148,13 @@ impl RecoverableJobHandler for FilesConsistencyCheck { FILES_CONSISTENCY_JOB_NAME } + fn description(&self) -> &'static str { + "Walks storage.files and reports rows whose parent-folder state, \ + blob reference or denormalised size has drifted from what the \ + join with folders and blobs says is true. Read-only — the fixes \ + live in other jobs (trash cascade, dedup_gc, blob resurrection)." + } + /// Definitive count — one row per file. This is the largest table /// of the trio (millions on big installs); COUNT(*) is still an /// index-only scan but can take ~seconds. The tradeoff is worth diff --git a/src/infrastructure/services/folders_consistency_service.rs b/src/infrastructure/services/folders_consistency_service.rs index 55ab99e0..18cb3e22 100644 --- a/src/infrastructure/services/folders_consistency_service.rs +++ b/src/infrastructure/services/folders_consistency_service.rs @@ -120,6 +120,14 @@ impl RecoverableJobHandler for FoldersConsistencyCheck { FOLDERS_CONSISTENCY_JOB_NAME } + fn description(&self) -> &'static str { + "Walks storage.folders and reports rows whose materialised path \ + and lpath have drifted from what walking the parent_id chain \ + produces. Any write path that bypasses the ltree cascade trigger \ + can leave these wrong, which silently breaks subtree queries. \ + Read-only." + } + /// Definitive count — one row per folder. Larger table than drives /// but the COUNT(*) is still index-only on PG. On multi-million-row /// deployments this is ~100ms at run start; acceptable given the diff --git a/src/infrastructure/services/grant_cleanup_service.rs b/src/infrastructure/services/grant_cleanup_service.rs index 32167b67..83925a95 100644 --- a/src/infrastructure/services/grant_cleanup_service.rs +++ b/src/infrastructure/services/grant_cleanup_service.rs @@ -23,7 +23,7 @@ use tracing::{error, info}; use crate::application::ports::authorization_ports::AuthorizationEngine; use crate::common::errors::DomainError; -use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs}; +use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs, Mutates}; use crate::infrastructure::services::pg_acl_engine::PgAclEngine; use async_trait::async_trait; @@ -122,6 +122,18 @@ impl JobHandler for GrantCleanupService { GRANT_CLEANUP_JOB_NAME } + fn description(&self) -> &'static str { + "Deletes expired role grants once they are past the retention \ + window. Expired grants never leak permission — every AuthZ check \ + filters on expires_at — they just accumulate. The window keeps \ + 'what happened to my access?' answerable for a few weeks after \ + expiry." + } + + fn mutates(&self) -> Mutates { + Mutates::Always + } + /// Runs one purge. `count` on the returned `JobOutcome::Ok` is /// the number of `role_grants` rows physically deleted; /// `extra.grace_days` records which grace was applied so admin diff --git a/src/infrastructure/services/image_transcode_service.rs b/src/infrastructure/services/image_transcode_service.rs index 4501f746..6b2e6a62 100644 --- a/src/infrastructure/services/image_transcode_service.rs +++ b/src/infrastructure/services/image_transcode_service.rs @@ -16,7 +16,7 @@ use bytes::Bytes; use image::ImageFormat; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use tokio::fs; @@ -112,6 +112,16 @@ struct AtomicTranscodeStats { transcodes: AtomicU64, bytes_saved: AtomicU64, transcode_errors: AtomicU64, + /// Decodes + encodes that produced something LARGER than the original. + /// + /// Counted separately because `transcodes` means "work that paid off" + /// — it is incremented only on the success path, alongside + /// `bytes_saved`. Without this counter the most expensive failure mode + /// is invisible: the full decode and re-encode of a multi-megapixel + /// image, repeated for every file sharing that content, producing + /// nothing. That is precisely the cost the persisted negative verdict + /// exists to eliminate, so it needs to be measurable before and after. + not_beneficial: AtomicU64, } /// Snapshot of transcoding statistics @@ -122,6 +132,7 @@ pub struct TranscodeStats { pub transcodes: u64, pub bytes_saved: u64, pub transcode_errors: u64, + pub not_beneficial: u64, } impl AtomicTranscodeStats { @@ -132,6 +143,7 @@ impl AtomicTranscodeStats { transcodes: self.transcodes.load(Ordering::Relaxed), bytes_saved: self.bytes_saved.load(Ordering::Relaxed), transcode_errors: self.transcode_errors.load(Ordering::Relaxed), + not_beneficial: self.not_beneficial.load(Ordering::Relaxed), } } } @@ -147,6 +159,29 @@ pub struct ImageTranscodeService { memory_cache: moka::future::Cache, /// Lock-free statistics stats: Arc, + /// The derived tier, attached after construction. + /// + /// A constructor parameter would be cleaner but does not fit: DI builds + /// this service before `DedupService` exists, and reordering is worse + /// than a one-shot — the transcode service is needed by the retrieval + /// path, which is wired early. `ThumbnailService` met the same wall and + /// took a per-call parameter instead; that does not work here because + /// the caller (`FileRetrievalService`) holds no dedup handle either, so + /// threading one through would push the dependency into a service that + /// has no other use for it. + /// + /// `OnceLock` rather than a `Mutex`: set exactly once at boot, read on + /// every request, never replaced. + dedup: OnceLock>, + /// Whether `.transcoded/` still exists, probed once by + /// [`Self::initialize`]. `false` short-circuits the local-cache reads + /// without a syscall. + /// + /// Starts `true` so a service constructed without `initialize` (tests) + /// behaves as before. Failing open is the safe direction: the wrong + /// value costs syscalls, the opposite would hide cached entries that + /// are still there. + legacy_cache: AtomicBool, } impl ImageTranscodeService { @@ -176,21 +211,110 @@ impl ImageTranscodeService { cache_dir, memory_cache, stats: Arc::new(AtomicTranscodeStats::default()), + dedup: OnceLock::new(), + legacy_cache: AtomicBool::new(true), } } - /// Initialize the service (create cache directories) + /// Attach the derived tier. Called once from DI, after `DedupService` + /// exists. Until then — and in tests that never call it — the service + /// behaves exactly as before, reading and writing only its local cache. + pub fn attach_dedup( + &self, + dedup: Arc, + ) { + if self.dedup.set(dedup).is_err() { + tracing::warn!( + target: "oxicloud::transcode", + "attach_dedup called twice — the first handle is kept" + ); + } + } + + /// The `content_derived_blobs.kind` for everything this service writes. + const DERIVED_KIND: &'static str = "transcode"; + + /// Memory-cache key: by CONTENT when the caller supplied a hash, by + /// file id only when it could not. + /// + /// Transcoding is a pure function of the source bytes, so file keying + /// was always the wrong axis for this cache — it just predated the + /// content-keyed tier. Two files with identical content held two + /// entries for identical bytes, and the second file missed RAM and + /// paid a DB lookup plus a blob read to fetch what was already in + /// memory under another key. + /// + /// The `c:` / `f:` prefixes keep the two namespaces disjoint. A + /// 64-hex hash and a UUID cannot collide in practice, but relying on + /// "in practice" for a cache key is how a file ends up served another + /// file's bytes. + /// + /// Same shape as `ThumbnailCacheKey`'s `content` / `external` split, + /// for the same reason: hash-less callers (external mounts) have no + /// content identity to key on, so they keep the per-file entry. + fn cache_key(source_hash: Option<&str>, file_id: &str, format: OutputFormat) -> String { + match source_hash { + Some(hash) => format!("c:{}:{}", hash, format.extension()), + None => format!("f:{}:{}", file_id, format.extension()), + } + } + + /// Initialize the service. + /// + /// Still creates the local cache directories, because this service DOES + /// still write them — unlike `ThumbnailService`, whose sidecar writes + /// are gone. When the transcode write path moves fully to the derived + /// tier, these two `create_dir_all` calls have to go at the same time: + /// leaving them would recreate the tree on every boot and make the + /// absence that `transcode_import` works toward unreachable, which is + /// exactly the bug that kept `.thumbnails/` alive across restarts. pub async fn initialize(&self) -> std::io::Result<()> { - fs::create_dir_all(&self.cache_dir).await?; - fs::create_dir_all(self.cache_dir.join("webp")).await?; + // Probes, does NOT create. + // + // Creating the tree at boot is what kept `.thumbnails/` alive across + // restarts: the import removed it, the next boot put it back, and + // the absence the read path gates on was unreachable by + // construction. The write path below already calls `create_dir_all` + // on the parent before writing, so nothing needs it created eagerly + // — the only thing eager creation achieved was defeating the drain. + // + // One `stat` on the root, cached for the process lifetime. It can + // only be stale in the harmless direction: a drain completing + // mid-life leaves the flag true until restart, costing the same + // failed opens as before. It never goes false while entries remain, + // because only `transcode_import` removes the tree and it removes + // the whole thing at once. + let present = fs::metadata(&self.cache_dir).await.is_ok(); + self.legacy_cache.store(present, Ordering::Relaxed); + tracing::info!( - "🖼️ Image transcode service initialized (rayon pool: {} threads, cache dir: {:?})", + "🖼️ Image transcode service initialized (rayon pool: {} threads)", transcode_thread_count(), - self.cache_dir ); + if present { + tracing::info!( + target: "oxicloud::transcode", + event = "transcode.legacy_cache_present", + path = ?self.cache_dir, + "legacy transcode cache present — reads fall back to it. Run \ + transcode_import with ?repair=true to drain it." + ); + } Ok(()) } + /// Whether the legacy local cache is worth touching. + /// + /// Unlike the thumbnail tiers this may legitimately never reach `false`: + /// callers with no content hash (external mounts) cannot use the + /// content-keyed tier at all, so they still read and write here. On an + /// install without such mounts the directory drains once and stays + /// gone; on one with them it persists, and that is correct rather than + /// a stalled migration. + fn legacy_cache_active(&self) -> bool { + self.legacy_cache.load(Ordering::Relaxed) + } + /// Check if a mime type can be transcoded. /// /// JPEG is deliberately excluded: the `image` crate's WebP encoder is @@ -213,14 +337,24 @@ impl ImageTranscodeService { /// /// Accepts `Bytes` (ref-counted) so callers avoid copying the buffer. /// Cloning `Bytes` is O(1) — only an atomic increment. + /// + /// `source_hash` is the BLAKE3 of the ORIGINAL content — the key the + /// derived tier uses. `None` falls back to the local cache alone, which + /// is what happens for callers that have no hash (external mounts) and + /// what the whole service did before the derived tier existed. + /// + /// It is a parameter rather than something computed here on purpose: + /// hashing `original_content` per request would be a BLAKE3 over the + /// whole file on every GET, and the caller already has the value. pub async fn get_transcoded( &self, file_id: &str, + source_hash: Option<&str>, original_content: Bytes, original_mime: &str, target_format: OutputFormat, ) -> Result<(Bytes, String, bool), String> { - let cache_key = format!("{}:{}", file_id, target_format.extension()); + let cache_key = Self::cache_key(source_hash, file_id, target_format); // ── 1. Fast path: moka memory cache (lock-free read) ── // An empty-Bytes entry is the negative sentinel: "transcoding this @@ -248,8 +382,14 @@ impl ImageTranscodeService { let cached = self .memory_cache .try_get_with(cache_key, async { - self.compute_transcode(file_id, original_for_loader, original_mime, target_format) - .await + self.compute_transcode( + file_id, + source_hash, + original_for_loader, + original_mime, + target_format, + ) + .await }) .await // try_get_with shares one `Arc` across waiters; DomainError @@ -271,13 +411,64 @@ impl ImageTranscodeService { async fn compute_transcode( &self, file_id: &str, + source_hash: Option<&str>, original_content: Bytes, original_mime: &str, target_format: OutputFormat, ) -> Result { - // ── Disk cache (async fs) ── + // ── Derived tier, ahead of the local cache ── + // + // Content-keyed, so it is shared across every file with these bytes + // and survives both a restart and a backend migration — neither of + // which the local `.transcoded/` tree does. Read first for the same + // reason the thumbnail read-order flip put it first: the local tree + // is the legacy tier being drained, and a fallback that is consulted + // first never stops being load-bearing. + let derived = match (source_hash, self.dedup.get()) { + (Some(hash), Some(dedup)) => Some((hash, dedup)), + _ => None, + }; + if let Some((hash, dedup)) = derived { + use crate::application::ports::dedup_ports::DerivedLookup; + match dedup + .lookup_derived(hash, Self::DERIVED_KIND, target_format.extension()) + .await + { + DerivedLookup::Found(r) => match dedup.read_blob_bytes(&r.blob_hash).await { + Ok(bytes) if !bytes.is_empty() => { + self.stats.disk_hits.fetch_add(1, Ordering::Relaxed); + tracing::debug!("🧱 Transcode derived tier HIT: {}", file_id); + return Ok(bytes); + } + // The row promised bytes that are gone or empty. Fall + // through and re-derive rather than serving nothing — + // a transcode is a pure function of its source, so this + // is recoverable by construction. `satellites_consistency` + // reports the dangling row separately. + _ => tracing::warn!( + target: "oxicloud::transcode", + source_hash = %hash, + blob_hash = %r.blob_hash, + "derived transcode row points at unreadable bytes; re-deriving" + ), + }, + // Known not worth transcoding for this content. This is the + // whole point of persisting the verdict: without it every GET + // repeats a full decode + encode to throw the result away. + DerivedLookup::NotDerivable => { + self.stats.disk_hits.fetch_add(1, Ordering::Relaxed); + tracing::debug!("🧱 Transcode negative derived row HIT: {}", file_id); + return Ok(Bytes::new()); + } + DerivedLookup::Missing => {} + } + } + + // ── Legacy local cache (async fs) ── + // + // Drained by `transcode_import`; kept as a fallback until it is gone. let cache_path = self.get_cache_path(file_id, target_format); - if tokio::fs::try_exists(&cache_path).await.unwrap_or(false) { + if self.legacy_cache_active() && tokio::fs::try_exists(&cache_path).await.unwrap_or(false) { match fs::read(&cache_path).await { Ok(data) => { self.stats.disk_hits.fetch_add(1, Ordering::Relaxed); @@ -292,7 +483,8 @@ impl ImageTranscodeService { // ── Negative verdict persisted on disk (survives restarts) ── let skip_marker = self.get_skip_marker_path(file_id, target_format); - if tokio::fs::try_exists(&skip_marker).await.unwrap_or(false) { + if self.legacy_cache_active() && tokio::fs::try_exists(&skip_marker).await.unwrap_or(false) + { self.stats.disk_hits.fetch_add(1, Ordering::Relaxed); tracing::debug!("💾 Transcode negative disk marker HIT: {}", file_id); return Ok(Bytes::new()); @@ -320,41 +512,127 @@ impl ImageTranscodeService { let transcoded_size = transcoded_bytes.len(); if transcoded_size >= original_size { + // Counted here, not with `transcodes` — the work happened but + // paid nothing, and conflating the two would hide the cost this + // whole negative-verdict mechanism exists to stop paying. + self.stats.not_beneficial.fetch_add(1, Ordering::Relaxed); tracing::debug!( "⚠️ Transcode not beneficial for {}: {} -> {} bytes", file_id, original_size, transcoded_size ); - // Remember the negative verdict so the next GET doesn't repeat the - // decode + encode: the caller caches the empty-Bytes sentinel (TTL) - // and we drop a zero-byte marker on disk (survives restarts; - // removed by `invalidate` when the file changes). - let marker = self.get_skip_marker_path(file_id, target_format); - tokio::spawn(async move { - if let Some(parent) = marker.parent() { - let _ = fs::create_dir_all(parent).await; + // Remember the verdict so the next GET does not repeat the decode + // + encode. The caller caches the empty-Bytes sentinel in RAM + // (10 min TTL); this row is what makes it survive eviction, a + // restart, and a move to another instance. + // + // Safe to persist because it is deterministic in the CONTENT: + // these exact bytes will always re-encode larger. A timeout or a + // read error would not be — those return `Err` above and are + // deliberately not recorded, since a momentary failure written + // here would mark a perfectly transcodable image as hopeless + // with nothing to ever retry it. + match derived { + Some((hash, dedup)) => { + if let Err(e) = dedup + .store_derived_negative(hash, Self::DERIVED_KIND, target_format.extension()) + .await + { + tracing::warn!( + target: "oxicloud::transcode", + source_hash = %hash, + error = %e, + "failed to persist negative transcode verdict; it will be recomputed" + ); + } } - if let Err(e) = fs::write(&marker, b"").await { - tracing::warn!("Failed to persist transcode skip marker: {}", e); + // Hash-less callers still get the zero-byte marker, for the + // same reason they still get the local cache write: the + // content-keyed tier cannot hold a verdict for content it + // cannot name. Dropping this would make every external-mount + // GET of a non-shrinking image re-decode once moka's TTL + // expires. + None => { + let marker = self.get_skip_marker_path(file_id, target_format); + tokio::spawn(async move { + if let Some(parent) = marker.parent() { + let _ = fs::create_dir_all(parent).await; + } + if let Err(e) = fs::write(&marker, b"").await { + tracing::warn!("Failed to persist transcode skip marker: {}", e); + } + }); } - }); + } return Ok(Bytes::new()); } let saved = original_size - transcoded_size; - // ── Persist to disk cache (fire-and-forget) ── - let cache_path_clone = cache_path.clone(); - let transcoded_for_disk = transcoded_bytes.clone(); - tokio::spawn(async move { - if let Some(parent) = cache_path_clone.parent() { - let _ = fs::create_dir_all(parent).await; + // ── Persist ── + // + // Derived tier when we have a source hash, local cache otherwise. + // Not both: writing the sidecar too would mean `transcode_import` + // chases a tail that keeps being refilled, which is the trap the + // thumbnail migration hit — four render paths wrote the sidecar and + // one wrote the row, so the tail never emptied. + // + // The local write survives only for hash-less callers (external + // mounts), which the derived tier cannot serve at all. When those + // gain a hash this branch goes, and `initialize`'s `create_dir_all` + // calls go with it. + match derived { + Some((hash, dedup)) => { + let dedup = dedup.clone(); + let hash = hash.to_string(); + let variant = target_format.extension().to_string(); + let mime = target_format.mime_type().to_string(); + let bytes = transcoded_bytes.clone(); + // Awaited, NOT spawned. + // + // Fire-and-forget looked free — the bytes are already on + // their way to the client — but it raced its own purpose. A + // second request for the SAME content arriving before the + // spawn lands finds no row, re-runs the whole decode + + // encode, and stores the identical blob again. The point of + // keying by content is that identical content is derived + // once; a write that has not landed yet cannot deliver + // that, and the window is milliseconds wide precisely when + // it matters most (a page loading many images at once). + // + // Caught by `transcode_cache.hurl`, which asserts the second + // distinct file with identical bytes does not re-transcode + // — it had been passing on timing luck. + // + // The cost is bounded: this path has just spent a full + // decode and re-encode, so one blob write is marginal + // beside it, and it only runs on a genuine miss. + if let Err(e) = dedup + .store_derived_blob(&hash, Self::DERIVED_KIND, &variant, &mime, bytes) + .await + { + tracing::warn!( + target: "oxicloud::transcode", + source_hash = %hash, + error = %e, + "failed to store derived transcode; it will be recomputed" + ); + } } - if let Err(e) = fs::write(&cache_path_clone, &transcoded_for_disk).await { - tracing::warn!("Failed to cache transcoded image: {}", e); + None => { + let cache_path_clone = cache_path.clone(); + let transcoded_for_disk = transcoded_bytes.clone(); + tokio::spawn(async move { + if let Some(parent) = cache_path_clone.parent() { + let _ = fs::create_dir_all(parent).await; + } + if let Err(e) = fs::write(&cache_path_clone, &transcoded_for_disk).await { + tracing::warn!("Failed to cache transcoded image: {}", e); + } + }); } - }); + } // ── Update stats (lock-free atomics) ── self.stats.transcodes.fetch_add(1, Ordering::Relaxed); @@ -392,7 +670,18 @@ impl ImageTranscodeService { /// Invalidate cached transcodes for a file pub async fn invalidate(&self, file_id: &str) { - let cache_key = format!("{}:{}", file_id, OutputFormat::WebP.extension()); + // Only the FILE-keyed entry, deliberately. + // + // Content-keyed entries must not be dropped here: this file's + // content changing says nothing about the other files sharing the + // old bytes, and evicting theirs would make one user's edit cost + // everyone else a re-transcode. They need no eviction anyway — + // new content is a new hash, so the old key is simply never + // consulted again, and moka's TTL reclaims it. + // + // What remains here is the fallback entry for hash-less callers, + // plus the legacy on-disk pair, which are genuinely per-file. + let cache_key = Self::cache_key(None, file_id, OutputFormat::WebP); self.memory_cache.invalidate(&cache_key).await; let cache_path = self.get_cache_path(file_id, OutputFormat::WebP); @@ -425,6 +714,65 @@ impl ImageTranscodeService { // ─── CPU-bound transcoding (runs on rayon, never on Tokio) ─────────────────── +#[cfg(test)] +mod fixture_premise { + //! Pins the property `tests/api/transcode_cache.hurl` is built on: one + //! fixture WebP shrinks, one it does not. + //! + //! The negative half was hard to come by and the reason is worth + //! recording. Synthetic images do not reproduce it — flat colour goes + //! 4780 → 186 bytes, a gradient 24852 → 102, and even uniform RGBA + //! noise still loses by ~242 bytes at any size, a margin that is + //! constant in absolute terms and so never flips. + //! + //! Two things have to be true at once, and only real content does + //! both. The encoder here is the `image` crate's own minimal VP8L + //! writer, not libwebp — it does none of libwebp's search over + //! predictors, colour transforms and Huffman groups — so it only wins + //! where redundancy is extreme enough that any encoder finds it. And + //! the original has to be near PNG-optimal, which a screenshot from a + //! real capture tool is: a 2× Retina UI is long identical runs, flat + //! panels and sharp edges, exactly what PNG's scanline filters plus + //! zlib were designed around. + //! + //! So the negative verdict this service persists is partly a property + //! of THIS encoder, not of the content. Swapping in libwebp would + //! likely flip most of these to positive and leave the stored negative + //! rows stale — an encoder change has to purge them. + + use super::*; + + fn webp_len(path: &str) -> (usize, usize) { + let png = std::fs::read(path).expect("fixture present"); + let webp = + transcode_image_blocking(&Bytes::from(png.clone()), "image/png", OutputFormat::WebP) + .expect("fixture decodes"); + (png.len(), webp.len()) + } + + /// If this ever fails, the hurl scenario's negative half has silently + /// become a second positive test — it would still pass while checking + /// nothing it was written to check. + #[test] + fn screenshot_fixture_is_a_genuine_negative() { + let (png, webp) = webp_len("tests/fixtures/negative-cache-transcode.png"); + assert!( + webp >= png, + "negative-cache-transcode.png no longer defeats the WebP encoder: \ + png={png} webp={webp}" + ); + } + + #[test] + fn flat_colour_fixture_is_a_genuine_positive() { + let (png, webp) = webp_len("tests/fixtures/red-image.png"); + assert!( + webp < png, + "red-image.png stopped shrinking: png={png} webp={webp}" + ); + } +} + /// Perform actual image transcoding. This is a pure CPU function — safe to call /// from `rayon::spawn` or `spawn_blocking`. fn transcode_image_blocking( @@ -476,12 +824,14 @@ impl ImageTranscodePort for ImageTranscodeService { async fn get_transcoded( &self, file_id: &str, + source_hash: Option<&str>, original_content: Bytes, original_mime: &str, target_format: PortOutputFormat, ) -> Result<(Bytes, String, bool), DomainError> { self.get_transcoded( file_id, + source_hash, original_content, original_mime, target_format.into(), diff --git a/src/infrastructure/services/jwt_service.rs b/src/infrastructure/services/jwt_service.rs index 9f8385a2..939a37f5 100644 --- a/src/infrastructure/services/jwt_service.rs +++ b/src/infrastructure/services/jwt_service.rs @@ -55,6 +55,17 @@ struct JwtClaims { /// Serialised as `{"cnf": {"jkt": "..."}}` to match RFC 9449. #[serde(skip_serializing_if = "Option::is_none")] pub cnf: Option, + /// OIDC-style `sid` claim (RFC 8417 §4.1) — carries the + /// `auth.sessions.id` this access token was minted for so the + /// auth middleware can stamp per-session liveness without a DB + /// round trip. `None` on tokens minted by pre-`sid` builds so + /// deserialisation stays backward-compatible during rollout. + /// Kept as `String` on the wire (Uuid parses at the port + /// boundary) so a malformed value fails at token-decode time + /// with a clear parse error instead of poisoning the field + /// silently. + #[serde(skip_serializing_if = "Option::is_none")] + pub sid: Option, } /// RFC 9449 §5 confirmation-key wrapper. Only the `jkt` member is @@ -72,6 +83,17 @@ impl From for TokenClaims { // signed always carries a UUID `sub`; nil is a safe sentinel the // middleware rejects. See benches/ROUND14.md §A3. let sub_id = uuid::Uuid::parse_str(&claims.sub).unwrap_or_else(|_| uuid::Uuid::nil()); + // Parse `sid` at the boundary — same amortization rationale + // as `sub_id` above, and gives us a clean `Option` in + // `TokenClaims`. A parse failure (mint-time bug or hand- + // crafted claim) drops the sid to `None`; the middleware + // then simply skips the stamp — token still authenticates. + // Legitimate tokens minted by this codebase always carry a + // valid Uuid, so this only masks external drift. + let sid = claims + .sid + .as_deref() + .and_then(|s| uuid::Uuid::parse_str(s).ok()); TokenClaims { sub_id, sub: claims.sub, @@ -82,6 +104,7 @@ impl From for TokenClaims { email: claims.email, role: claims.role, dpop_jkt: claims.cnf.map(|c| c.jkt), + sid, } } } @@ -196,6 +219,7 @@ impl TokenServicePort for JwtTokenService { fn generate_access_token( &self, user: &User, + session_id: Option, dpop_jkt: Option<&str>, ) -> Result { let now = Utc::now().timestamp(); @@ -219,6 +243,7 @@ impl TokenServicePort for JwtTokenService { cnf: dpop_jkt.map(|jkt| CnfClaim { jkt: jkt.to_string(), }), + sid: session_id.map(|id| id.to_string()), }; // Log JWT claims for debugging @@ -331,7 +356,7 @@ mod tests { let user = create_test_user(); let token = service - .generate_access_token(&user, None) + .generate_access_token(&user, Some(Uuid::new_v4()), None) .expect("Should generate token"); let claims = service @@ -370,7 +395,7 @@ mod tests { let user = create_test_user(); let token = service - .generate_access_token(&user, None) + .generate_access_token(&user, Some(Uuid::new_v4()), None) .expect("Should generate token"); // First call: cache miss — performs full HMAC verification @@ -397,7 +422,7 @@ mod tests { 86400, ); let token = service - .generate_access_token(&create_test_user(), None) + .generate_access_token(&create_test_user(), Some(Uuid::new_v4()), None) .expect("Should generate token"); // Miss populates the cache; hit must hand back the very same @@ -425,4 +450,46 @@ mod tests { let (hits, _misses) = service.cache_stats(); assert_eq!(hits, 0, "Invalid tokens should never produce cache hits"); } + + /// Regression for the `sid` claim wiring — the auth middleware + /// stamps per-session liveness by reading this exact field. If + /// the mint stops setting the claim or the port stops parsing + /// it, every `LastSeenTracker::stamp` call goes silent and the + /// Prometheus gauges freeze at zero. + #[test] + fn access_token_round_trips_session_id_as_sid_claim() { + let service = JwtTokenService::new( + "test_secret_key_at_least_32_bytes_long".to_string(), + 3600, + 86400, + ); + let user = create_test_user(); + let session_id = Uuid::new_v4(); + let token = service + .generate_access_token(&user, Some(session_id), None) + .expect("Should generate token"); + let claims = service.validate_token(&token).expect("Should validate"); + assert_eq!(claims.sid, Some(session_id)); + } + + /// Backward-compatibility guard: a mint call with `None` + /// omits the `sid` claim entirely (matches the pre-`sid` + /// on-wire shape), and the validated claims surface `None` + /// on the port. The middleware's `if let (Some(sid), ...)` + /// then simply skips the stamp — critical during rollout + /// where old tokens are still in flight. + #[test] + fn access_token_without_session_id_omits_sid_claim() { + let service = JwtTokenService::new( + "test_secret_key_at_least_32_bytes_long".to_string(), + 3600, + 86400, + ); + let user = create_test_user(); + let token = service + .generate_access_token(&user, None, None) + .expect("Should generate token"); + let claims = service.validate_token(&token).expect("Should validate"); + assert_eq!(claims.sid, None); + } } diff --git a/src/infrastructure/services/last_seen_tracker.rs b/src/infrastructure/services/last_seen_tracker.rs new file mode 100644 index 00000000..577b8a05 --- /dev/null +++ b/src/infrastructure/services/last_seen_tracker.rs @@ -0,0 +1,239 @@ +//! Per-session liveness tracker — the hot path of the "how many +//! sessions are active right now?" observation loop. +//! +//! **Contract.** Every authenticated request calls +//! [`LastSeenTracker::stamp`] with the session id it resolved. The +//! call is O(1) — a DashMap upsert of `(session_id → Utc::now())` — +//! and hits no I/O. The map data structure IS the dedup: 100 +//! requests against the same session in a flush window contribute +//! ONE row to the batched UPDATE with the latest timestamp. +//! +//! A background task ([`flush_loop`](Self::flush_loop), spawned at +//! construction) drains the map every 30 s and issues one +//! `UPDATE ... FROM UNNEST($1::uuid[], $2::timestamptz[])` covering +//! every distinct session_id observed in the window. The +//! `greatest(s.last_seen_at, t.seen_at)` guard makes the write +//! idempotent under any retry / race / clock skew — replaying the +//! same batch never moves the column backward. +//! +//! **Failure model.** A flush that hits a transient PG error does +//! NOT drop the accumulated set — the map is not cleared until the +//! UPDATE succeeds. Next tick overlays new activity on the retry +//! set and the whole thing gets flushed together. Bounded loss +//! window under a hard crash is one flush interval; graceful +//! shutdown calls [`flush_now`](Self::flush_now) synchronously (see +//! `main.rs`) so rolling restarts drop nothing. +//! +//! **Non-goals.** No per-session locking, no ordering guarantees +//! across sessions, no back-pressure on the flusher (the loop +//! swallows errors and keeps ticking). The workload is +//! observation-only — losing a stamp under contention is a +//! correctness no-op, the next request re-stamps. +//! +//! See `docs/plan/sessions.md` for the full design (why DashMap +//! over Mutex, why not NOTIFY/LISTEN today, migration +//! path to a multi-instance cluster). + +use std::sync::Arc; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use sqlx::PgPool; +use uuid::Uuid; + +/// Cadence of the batched UPDATE. Hardcoded — 30 s balances DB +/// write load against gauge freshness (the Prometheus scrape +/// interval is typically 15 s, so at worst two scrapes see the +/// same value before the next flush). Deliberately NOT exposed as +/// an env var — tuning it is a deployment-shape question we've +/// never had to answer in practice. +const FLUSH_INTERVAL: Duration = Duration::from_secs(30); + +/// In-process session-liveness tracker. See [module docs](self) for +/// the full contract; the two entry points are: +/// +/// - [`stamp`](Self::stamp) — called from the auth middleware on +/// every authenticated request. +/// - [`flush_now`](Self::flush_now) — called from the graceful- +/// shutdown handler. +/// +/// The periodic flush task is spawned on the tokio runtime by +/// [`start`](Self::start) at construction. The struct keeps no +/// handle to it — the task holds the `Arc` and observes the +/// runtime shutting down naturally. +pub struct LastSeenTracker { + /// (session_id → last observed time). DashMap's sharded locking + /// parallelises writes across distinct session_ids — different + /// users' requests never contend. + seen: DashMap>, + /// Maintenance pool — the tracker is a background writer and + /// must not compete with request-serving connections. + pool: Arc, +} + +impl LastSeenTracker { + /// Construct + spawn the flush loop. Returns the shared + /// handle; callers store it on `AppState` and pass it to the + /// auth middleware. + /// + /// The background task lives for the runtime's lifetime — no + /// cancellation handle is exposed because there is no + /// mid-process reason to stop tracking (a stopped flusher is + /// indistinguishable from a wedged one, and both are bugs). + /// Graceful shutdown calls [`flush_now`](Self::flush_now) + /// separately BEFORE the runtime tears down. + pub fn start(pool: Arc) -> Arc { + let this = Arc::new(Self { + seen: DashMap::new(), + pool, + }); + tokio::spawn(this.clone().flush_loop()); + this + } + + /// Record that `session_id` was observed serving a request + /// right now. Overwrites any prior stamp for the same session + /// in the current window — the flusher uses the latest value. + /// + /// O(1) DashMap upsert. No I/O. Never fails. + pub fn stamp(&self, session_id: Uuid) { + self.seen.insert(session_id, Utc::now()); + } + + /// Drain the accumulated stamps and write them in one batched + /// UPDATE. Idempotent — the `greatest(...)` guard means + /// replaying the same batch (or overlapping batches from a + /// retry) never moves the column backward. + /// + /// Errors are surfaced to the caller so `flush_loop`'s + /// warn-and-continue policy is a deliberate choice made in one + /// place, and the shutdown flusher in `main.rs` can decide + /// whether to log or panic. + /// + /// On PG error the accumulated set is NOT cleared — the next + /// tick retries with fresh activity overlaid. + pub async fn flush_now(&self) -> Result { + if self.seen.is_empty() { + return Ok(0); + } + + // Drain into two parallel vectors — one UNNEST arg each. + // `retain(|_,_| false)` clears every shard in-place; the + // pull-and-drop order doesn't matter (we upserted the + // latest wins per key already). + let mut ids: Vec = Vec::with_capacity(self.seen.len()); + let mut seen_at: Vec> = Vec::with_capacity(self.seen.len()); + for entry in self.seen.iter() { + ids.push(*entry.key()); + seen_at.push(*entry.value()); + } + + let result = sqlx::query( + r#" + UPDATE auth.sessions AS s + SET last_seen_at = greatest(s.last_seen_at, t.seen_at) + FROM UNNEST($1::uuid[], $2::timestamptz[]) AS t(id, seen_at) + WHERE s.id = t.id + "#, + ) + .bind(&ids) + .bind(&seen_at) + .execute(&*self.pool) + .await?; + + // Only clear the drained keys on success. A key inserted + // BETWEEN our copy above and the clear below survives + // (retain drops only those whose value we already flushed, + // by timestamp equality). Same-key re-stamp with a newer + // timestamp gets kept for the next flush. + let flushed: std::collections::HashMap> = + ids.iter().copied().zip(seen_at.iter().copied()).collect(); + self.seen + .retain(|k, v| flushed.get(k).is_none_or(|ts| ts != v)); + + let updated = result.rows_affected() as usize; + tracing::debug!( + target: "oxicloud::sessions", + batched = ids.len(), + updated, + "last_seen flush", + ); + Ok(updated) + } + + /// The periodic drain loop. Runs forever; every failed flush + /// is logged at WARN and the accumulated set is preserved for + /// the next tick. + async fn flush_loop(self: Arc) { + let mut ticker = tokio::time::interval(FLUSH_INTERVAL); + // Skip the "first tick fires immediately" behaviour — the + // map is empty at spawn time, so a same-tick flush is + // wasted work. + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + ticker.tick().await; + + loop { + ticker.tick().await; + if let Err(err) = self.flush_now().await { + tracing::warn!( + target: "oxicloud::sessions", + error = %err, + "last_seen flush failed; will retry next tick", + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Ten stamps of the same session_id must collapse to ONE + /// entry with the newest timestamp — the whole point of the + /// DashMap-as-dedup pattern. Guards against a future refactor + /// that swaps to an append-only channel and doubles the DB + /// write rate. + #[test] + fn stamps_dedup_by_session_id() { + // No pool needed — we're only exercising the map. Build + // the tracker directly without spawning the loop. + let seen = DashMap::new(); + let session = Uuid::new_v4(); + + for _ in 0..10 { + seen.insert(session, Utc::now()); + } + + assert_eq!(seen.len(), 1); + } + + /// Latest-wins semantics: two stamps for the same session + /// leave the newer timestamp in place, matching the flusher's + /// `greatest(...)` guard so a request that beats the flush + /// keeps its more recent stamp. + #[test] + fn stamp_keeps_latest_timestamp() { + let seen: DashMap> = DashMap::new(); + let session = Uuid::new_v4(); + + let t1 = Utc::now(); + seen.insert(session, t1); + let t2 = t1 + chrono::Duration::seconds(5); + seen.insert(session, t2); + + assert_eq!(*seen.get(&session).unwrap(), t2); + } + + /// Distinct sessions never collide — sharded map, no dedup + /// across keys. + #[test] + fn different_sessions_are_independent() { + let seen: DashMap> = DashMap::new(); + for _ in 0..100 { + seen.insert(Uuid::new_v4(), Utc::now()); + } + assert_eq!(seen.len(), 100); + } +} diff --git a/src/infrastructure/services/local_blob_backend.rs b/src/infrastructure/services/local_blob_backend.rs index 20ed4eaf..4f0fac15 100644 --- a/src/infrastructure/services/local_blob_backend.rs +++ b/src/infrastructure/services/local_blob_backend.rs @@ -778,12 +778,30 @@ impl BlobStorageBackend for LocalBlobBackend { let blob_root = self.blob_root.clone(); Box::pin(async move { + // Cursor is the last hash returned (see the port contract). The + // shard is derivable from it — the shard name IS the hash's first + // two chars — so no composite is needed. + // + // Both legacy forms still resume correctly, so a consistency run + // paused across this deploy is not stranded: + // * "/" — what this backend used to emit; the + // hash half is taken and the shard re-derived from it. + // * "" — a bare 2-char shard. It flows through the same + // path: "3f" sorts BEFORE every 64-char hash beginning "3f", + // so using it as start_after skips nothing. let (start_shard, start_after_hash): (String, Option) = match cursor { None => (String::from("00"), None), - Some(c) => match c.split_once('/') { - Some((sh, h)) => (sh.to_string(), Some(h.to_string())), - None => (c, None), - }, + Some(c) => { + let hash = c.split_once('/').map(|(_, h)| h).unwrap_or(c.as_str()); + if hash.len() >= 2 { + (hash[..2].to_string(), Some(hash.to_string())) + } else { + // Under 2 chars — not a hash and not a shard. Should + // be unreachable; start from the beginning rather + // than index out of bounds. + (String::from("00"), None) + } + } }; let mut blobs: Vec = Vec::with_capacity(limit); @@ -879,11 +897,8 @@ impl BlobStorageBackend for LocalBlobBackend { continue; } if blobs.len() >= limit { - next_cursor = Some(format!( - "{}/{}", - prefix, - blobs.last().map(|e| e.hash.as_str()).unwrap_or("") - )); + // Just the hash — the shard is recoverable from it. + next_cursor = blobs.last().map(|e| e.hash.clone()); return Ok(BlobListPage { blobs, unknowns, @@ -1008,4 +1023,66 @@ mod tests { ); assert_eq!(hash_prefix_slot("gg"), None); } + + /// The port contract now REQUIRES ascending hash order and a cursor that + /// is the last hash returned. `backend_consistency`'s merge-join depends + /// on both: an out-of-order page would make it emit bogus + /// `blob_missing_from_backend` findings at `data_loss` severity, and a + /// non-hash cursor would stop a caller resuming from its own checkpoint. + /// + /// Nothing covered enumeration before this, so both properties were + /// accidental. + #[tokio::test] + async fn list_blob_hashes_is_ordered_and_hash_cursor_resumes() { + let dir = TempDir::new().unwrap(); + let backend = LocalBlobBackend::new(dir.path()); + backend.initialize().await.unwrap(); + + // Deliberately inserted out of order and across several shards, so a + // passing result cannot come from insertion order. + let mut written: Vec = ["f0", "0a", "9c", "0b", "ff", "12"] + .iter() + .map(|p| fake_hash(p)) + .collect(); + for h in &written { + backend + .put_blob_from_bytes(h, Bytes::from_static(b"x")) + .await + .unwrap(); + } + written.sort(); + + // Page with limit 2 so the cursor is exercised repeatedly. + let mut seen: Vec = Vec::new(); + let mut cursor: Option = None; + for _ in 0..20 { + let page = backend.list_blob_hashes(cursor.clone(), 2).await.unwrap(); + seen.extend(page.blobs.iter().map(|e| e.hash.clone())); + match page.next_cursor { + Some(c) => cursor = Some(c), + None => break, + } + } + + assert_eq!(seen, written, "enumeration must be complete and ascending"); + + // A cursor the CALLER synthesises from a hash it already holds must + // work — that is the property the merge-join resume relies on, and + // what an opaque backend token could not provide. + let midpoint = &written[2]; + let resumed = backend + .list_blob_hashes(Some(midpoint.clone()), 100) + .await + .unwrap(); + let expected: Vec = written[3..].to_vec(); + assert_eq!( + resumed + .blobs + .iter() + .map(|e| e.hash.clone()) + .collect::>(), + expected, + "resume must start STRICTLY after the given hash" + ); + } } diff --git a/src/infrastructure/services/manifests_consistency_service.rs b/src/infrastructure/services/manifests_consistency_service.rs new file mode 100644 index 00000000..ddd9fe08 --- /dev/null +++ b/src/infrastructure/services/manifests_consistency_service.rs @@ -0,0 +1,566 @@ +//! Reconciles `storage.chunk_manifests.ref_count` against its actual +//! referrers. +//! +//! ### Why this exists +//! +//! There are **two** reference counters, and only one of them was ever +//! verified. `DedupService::add_reference` bumps +//! `chunk_manifests.ref_count` first and only falls back to +//! `storage.blobs.ref_count`, so a reference lands on whichever counter +//! its hash names: +//! +//! * a **chunk** reference → `storage.blobs.ref_count`, reconciled by +//! `blobs_consistency::refcount_mismatch`; +//! * a **Blob** reference (a CDC file, and now every derived artifact) → +//! `chunk_manifests.ref_count`, reconciled by **nothing** before this +//! job existed. +//! +//! That gap was survivable only because `dedup_gc`'s reap predicate had a +//! second clause — "no `storage.files` row references this manifest" — +//! which quietly compensated for drift on the bulk-delete paths where +//! `ref_count` is never decremented. Generalising that clause to the +//! reference registry (so thumbnails stop being reaped) removes the +//! compensation, which is exactly why the manifest counter now has to be +//! checked directly. See `docs/plan/derived-blobs.md`. +//! +//! ### The check +//! +//! * `manifest_refcount_mismatch` (severity `inconsistent`) — +//! `chunk_manifests.ref_count` disagrees with the number of registered +//! referrers. An **under**-count is the dangerous direction: GC reaps a +//! manifest whose content is still reachable, taking its chunks with it. +//! An over-count merely pins storage. Content-safe to report either way +//! — the manifest row and its chunks are intact, the counter is wrong. +//! +//! ### Why a separate job rather than a phase of `blobs_consistency` +//! +//! One subject per job, per the subject-iteration principle the other five +//! consistency tenants follow. It also avoids changing the cursor format of +//! an existing *recoverable* job, which would strand any run paused across +//! the deploy. + +use std::sync::Arc; + +use async_trait::async_trait; +use sqlx::PgPool; + +use crate::application::ports::blob_reference_ports::{BlobReferenceRegistry, RefLevel}; +use crate::infrastructure::scheduler::{ + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, Mutates, RecoverableJobHandler, + RunOutcome, RunStatus, record_or_log, +}; + +pub const MANIFESTS_CONSISTENCY_JOB_NAME: &str = "manifests_consistency"; + +/// Rows per batch. Each row costs one indexed subquery per registered +/// source; 200 matches `blobs_consistency` so the cancel-poll cadence is +/// the same for an operator watching either job. +const BATCH_SIZE: i64 = 200; + +/// The page query, with `actual_ref_count` summed from the registered +/// reference sources at [`RefLevel::Manifest`]. +/// +/// Only sources that reference a **Blob** contribute — `storage.files` +/// today, plus `storage.content_derived_blobs` and +/// `storage.file_attached_blobs` once they exist. +/// `ChunksReferenceSource` returns `None` here: a manifest is never +/// referenced by another manifest, and including it would count this +/// manifest's own chunks as referrers of itself. +/// +/// # Panics +/// +/// If no source contributes at [`RefLevel::Manifest`] — a wiring bug that +/// would report every manifest as mismatched. +fn manifest_page_sql(registry: &BlobReferenceRegistry) -> String { + let expected = registry.ref_count_expr(RefLevel::Manifest, "m.file_hash"); + assert!( + expected != "0", + "no manifest-level blob reference source registered: every manifest \ + would appear unreferenced" + ); + + format!( + "SELECT + m.file_hash AS file_hash, + m.ref_count AS ref_count, + m.total_size AS total_size, + m.chunk_count AS chunk_count, + ({expected})::bigint AS actual_ref_count + FROM storage.chunk_manifests m + WHERE ($1::text IS NULL OR m.file_hash > $1) + ORDER BY m.file_hash + LIMIT $2" + ) +} + +/// Repair statement targeting one manifest by `file_hash`. Uses the +/// SAME registry-derived expression as [`manifest_page_sql`] so +/// detection and repair agree on what "actual" means — any future +/// manifest-level ref source added to the registry flows into both +/// queries with no code change here. +/// +/// The `<> (subquery)` guard makes the UPDATE a no-op when the value +/// is already correct — so this is idempotent under concurrent-repair +/// races AND under retry. +/// +/// The subquery re-reads inside the same statement, so a concurrent +/// insert/delete between page fetch and this UPDATE can't leave a +/// stale value: PG's snapshot for the UPDATE sees the up-to-date row +/// counts. +fn manifest_repair_sql(registry: &BlobReferenceRegistry) -> String { + let expected = registry.ref_count_expr(RefLevel::Manifest, "m.file_hash"); + format!( + "UPDATE storage.chunk_manifests m + SET ref_count = ({expected})::bigint + WHERE m.file_hash = $1 + AND m.ref_count <> ({expected})::bigint" + ) +} + +pub struct ManifestsConsistencyCheck { + pool: Arc, + /// Built once from the blob-reference registry so this recompute and + /// `dedup_gc`'s reap predicate answer "what references this manifest" + /// identically. Assembled at construction rather than per page so the + /// sweep runs a fixed statement. + page_sql: String, + /// Repair statement — built from the SAME registry as `page_sql` so + /// detection and repair use identical formulas by construction. Any + /// future 4th manifest-level ref source added to the registry + /// automatically flows into both queries with no code change here. + /// + /// Previously the repair query was inlined with the files-only + /// formula, which meant drift from `content_derived_blobs` or + /// `file_attached_blobs` would be DETECTED but NOT repaired even + /// under `?repair=true`. Operators who added those tables saw + /// findings that couldn't be cleared by the repair path — bug fixed + /// 2026-09-02. + /// + /// The `?repair=true` gate on the trigger endpoint still stands as + /// the operator's explicit opt-in — this fix only widens what + /// repair CAN do when the operator chooses to run it. Discovery- + /// only remains the default so leaks in insert paths still surface + /// via findings between repair invocations. + repair_sql: String, +} + +impl ManifestsConsistencyCheck { + pub fn new(pool: Arc, reference_registry: Arc) -> Self { + Self { + pool, + page_sql: manifest_page_sql(&reference_registry), + repair_sql: manifest_repair_sql(&reference_registry), + } + } + + /// Chainable self-registration. On-demand only — operators fire it + /// from `POST /api/admin/jobs/manifests_consistency/trigger`. + pub async fn register_recoverable_job( + self: Arc, + registry: &JobRegistry, + provider: &Arc, + ) -> Arc { + registry + .register_recoverable_job(self.clone(), provider.clone(), None) + .await; + self + } +} + +#[derive(Debug, sqlx::FromRow)] +struct ManifestRow { + file_hash: String, + ref_count: i32, + total_size: i64, + chunk_count: i32, + actual_ref_count: i64, +} + +#[async_trait] +impl RecoverableJobHandler for ManifestsConsistencyCheck { + fn name(&self) -> &str { + MANIFESTS_CONSISTENCY_JOB_NAME + } + + fn description(&self) -> &'static str { + "Reconciles storage.chunk_manifests.ref_count against its actual \ + referrers. There are two reference counters — a chunk reference \ + lands on storage.blobs.ref_count, a whole-Blob reference on the \ + manifest — and only the first was ever verified; this covers the \ + other half." + } + + fn mutates(&self) -> Mutates { + Mutates::OnRepairOnly + } + + fn repair_description(&self) -> Option<&'static str> { + Some( + "Rewrites drifted manifest ref_count values to the recomputed \ + truth. Nothing is deleted here — a corrected count only makes \ + the manifest eligible for a later dedup_gc sweep.", + ) + } + + async fn count_total(&self) -> Option { + let row: Result<(i64,), sqlx::Error> = + sqlx::query_as("SELECT COUNT(*) FROM storage.chunk_manifests") + .fetch_one(self.pool.as_ref()) + .await; + match row { + Ok((n,)) => Some(n.max(0) as u64), + Err(e) => { + tracing::debug!( + target: "oxicloud::consistency", + event = "manifests_consistency.count_total_failed", + error = %e, + "count_total failed — run will not surface a progress bar" + ); + None + } + } + } + + async fn run_resumable( + &self, + store: &dyn JobStore, + args: &JobRunArgs, + resume_cursor: Option>, + ) -> RunOutcome { + let is_fresh = resume_cursor.is_none(); + + // Cursor: the last `file_hash` as UTF-8. Same convention as + // `blobs_consistency`, which also pages a hash-keyed table. + let mut cursor: Option = match resume_cursor { + None => None, + Some(bytes) if bytes.is_empty() => None, + Some(bytes) => match String::from_utf8(bytes) { + Ok(s) => Some(s), + Err(e) => { + return RunOutcome::Failed { + message: format!("invalid cursor: not valid UTF-8: {e}"), + }; + } + }, + }; + + // Persist the repair flag into `params.repair` so the admin + // run-detail view can display whether the run was a discovery + // scan or an active repair. Fresh takes it from args; Resume + // reads back so a paused repair scan stays a repair scan (a + // mid-scan crash mustn't silently downgrade the remaining + // rows to discovery-only). Same shape as + // `blobs_consistency_service.rs`'s `deep` handling — see the + // reasoning documented there. + let repair = if is_fresh { + let v = if args.repair { "true" } else { "false" }; + if let Err(e) = store.set_string_param("repair", v).await { + return RunOutcome::Failed { + message: format!("failed to persist repair flag to params: {e}"), + }; + } + args.repair + } else { + match store.get_string_param("repair").await { + Ok(Some(v)) => v == "true", + Ok(None) => false, + Err(e) => { + return RunOutcome::Failed { + message: format!("read `repair` from params: {e}"), + }; + } + } + }; + + if repair { + tracing::info!( + target: "oxicloud::consistency", + event = "manifests_consistency.repair_mode_active", + run_id = %store.run_id(), + "repair mode: manifest_refcount_mismatch findings will trigger corrective UPDATE" + ); + } + + let mut finding_count = 0u64; + // Only relevant when `repair == true`. Reported inline in + // the completion log + the `extra_stats` payload so operators + // can see "we found N and fixed M" in one line. + let mut repaired_count = 0u64; + + loop { + // Cooperative cancel poll between batches. + match store.status().await { + Ok(RunStatus::CancelRequested) => { + tracing::info!( + target: "oxicloud::consistency", + event = "manifests_consistency.cancelled", + run_id = %store.run_id(), + finding_count = finding_count, + "manifests_consistency cancelled cooperatively, pausing" + ); + return RunOutcome::Paused { + cursor: cursor + .as_ref() + .map(|s| s.as_bytes().to_vec()) + .unwrap_or_default(), + }; + } + Ok(_) => {} + Err(e) => { + return RunOutcome::Failed { + message: format!("status poll: {e}"), + }; + } + } + + let rows: Vec = match sqlx::query_as(&self.page_sql) + .bind(cursor.as_deref()) + .bind(BATCH_SIZE) + .fetch_all(self.pool.as_ref()) + .await + { + Ok(r) => r, + Err(e) => { + return RunOutcome::Failed { + message: format!("batch fetch: {e}"), + }; + } + }; + + if rows.is_empty() { + tracing::info!( + target: "oxicloud::consistency", + event = "manifests_consistency.completed", + run_id = %store.run_id(), + finding_count = finding_count, + repaired_count = repaired_count, + repair_requested = repair, + "manifests_consistency completed with {} finding(s), {} repaired", + finding_count, + repaired_count + ); + return RunOutcome::completed_with(serde_json::json!({ + "repair_requested": repair, + "repaired_count": repaired_count, + })); + } + + for row in &rows { + if row.ref_count as i64 == row.actual_ref_count { + continue; + } + finding_count += 1; + let delta = row.actual_ref_count - row.ref_count as i64; + let detail = serde_json::json!({ + "file_hash": row.file_hash, + "stored": row.ref_count, + "actual": row.actual_ref_count, + "delta": delta, + "total_size": row.total_size, + "chunk_count": row.chunk_count, + // Under-count is the dangerous direction: GC reaps + // a manifest whose content is still reachable. + "reap_risk": delta > 0, + }); + + // Repair pass — content-safe corrective UPDATE. The + // stored counter is set to what the auditor formula + // would compute at UPDATE time (subquery matches + // `manifest_page_sql`'s `actual_ref_count` predicate), + // so a concurrent file insert/delete between our page + // fetch and this UPDATE can't leave a stale value — + // the subquery re-reads inside the same statement. + // The `<> (subquery)` guard makes the UPDATE a no-op + // if the value is already correct, so this is + // idempotent under retry. + // + // `self.repair_sql` is built once at construction from + // the same `BlobReferenceRegistry` as the page query — + // detection and repair use identical formulas by + // construction. See `manifest_repair_sql` for the SQL. + // + // Attempt repair FIRST, then record the finding with + // severity/kind reflecting the final state: + // * repair succeeded → severity "info", kind "manifest_refcount_repaired" + // * repair no-op → severity "info", kind "manifest_refcount_resolved" + // * repair failed → severity "inconsistent", kind "manifest_refcount_mismatch" + // * no repair requested → severity "inconsistent", kind "manifest_refcount_mismatch" + // + // Parallels the WARN-then-INFO sequence in logs: an + // unresolved drift raises attention ("inconsistent"), + // a repaired one records the fix at info level without + // inflating the "needs action" tally the outcome UI + // shows. The detail JSON still carries `stored/actual/ + // delta` so the audit trail is complete either way. + let (kind, severity) = if repair { + match sqlx::query(&self.repair_sql) + .bind(&row.file_hash) + .execute(self.pool.as_ref()) + .await + { + Ok(res) if res.rows_affected() > 0 => { + repaired_count += 1; + tracing::info!( + target: "audit", + event = "manifests_consistency.repaired", + run_id = %store.run_id(), + file_hash = %row.file_hash, + stored_was = row.ref_count, + actual = row.actual_ref_count, + "🩹 manifest ref_count repaired" + ); + ("manifest_refcount_repaired", "info") + } + Ok(_) => { + // Row not touched — either another + // concurrent repair fixed it first, or the + // drift healed itself between page fetch + // and UPDATE. Either way, current state + // is correct — record as info. + ("manifest_refcount_resolved", "info") + } + Err(e) => { + tracing::warn!( + target: "oxicloud::consistency", + event = "manifests_consistency.repair_failed", + run_id = %store.run_id(), + file_hash = %row.file_hash, + error = %e, + "manifest ref_count repair UPDATE failed — finding stays" + ); + ("manifest_refcount_mismatch", "inconsistent") + } + } + } else { + ("manifest_refcount_mismatch", "inconsistent") + }; + + record_or_log( + store, + MANIFESTS_CONSISTENCY_JOB_NAME, + kind, + severity, + None, // a hash isn't a UUID; the identifier lives in detail + detail, + ) + .await; + } + + // Advance cursor + checkpoint. + let last_hash = rows + .last() + .map(|r| r.file_hash.clone()) + .expect("non-empty rows"); + cursor = Some(last_hash.clone()); + let batch_len = rows.len() as u64; + if let Err(e) = store.checkpoint(last_hash.into_bytes(), batch_len).await { + return RunOutcome::Failed { + message: format!("checkpoint: {e}"), + }; + } + + if (rows.len() as i64) < BATCH_SIZE { + tracing::info!( + target: "oxicloud::consistency", + event = "manifests_consistency.completed", + run_id = %store.run_id(), + finding_count = finding_count, + repaired_count = repaired_count, + repair_requested = repair, + "manifests_consistency completed with {} finding(s), {} repaired", + finding_count, + repaired_count + ); + return RunOutcome::completed_with(serde_json::json!({ + "repair_requested": repair, + "repaired_count": repaired_count, + })); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn default_registry() -> BlobReferenceRegistry { + let pool = Arc::new( + sqlx::pool::PoolOptions::::new() + .connect_lazy("postgres://invalid/invalid") + .expect("lazy pool never connects"), + ); + crate::infrastructure::repositories::pg::blob_reference_sources::built_in_registry(pool) + } + + /// Golden test — the statement is assembled from the registry, so pin it + /// byte-for-byte and read the SQL here rather than deriving it mentally. + /// + /// Two invariants a future source must not break: the files term carries + /// **no** `NOT EXISTS` guard (that guard exists to keep CDC rows out of + /// the *chunk* level; applying it here would count nothing), and + /// `chunk_hashes` appears nowhere — a manifest citing its own chunks is + /// not a referrer of itself. + #[tokio::test] + async fn manifest_page_statement_is_stable() { + let sql = manifest_page_sql(&default_registry()); + let expected = r#"SELECT + m.file_hash AS file_hash, + m.ref_count AS ref_count, + m.total_size AS total_size, + m.chunk_count AS chunk_count, + ((SELECT COUNT(*) FROM storage.files cnt_f + WHERE cnt_f.blob_hash = m.file_hash) + + (SELECT COUNT(*) FROM storage.content_derived_blobs cnt_d WHERE cnt_d.blob_hash = m.file_hash) + + (SELECT COUNT(*) FROM storage.file_attached_blobs cnt_a WHERE cnt_a.blob_hash = m.file_hash))::bigint AS actual_ref_count + FROM storage.chunk_manifests m + WHERE ($1::text IS NULL OR m.file_hash > $1) + ORDER BY m.file_hash + LIMIT $2"#; + assert_eq!(sql, expected, "manifest page statement changed:\n{sql}"); + } + + #[tokio::test] + async fn chunks_source_contributes_nothing_at_manifest_level() { + let sql = manifest_page_sql(&default_registry()); + assert!( + !sql.contains("chunk_hashes"), + "a manifest must not count its own chunks as referrers: {sql}" + ); + } + + #[test] + #[should_panic(expected = "no manifest-level blob reference source")] + fn empty_registry_refuses_to_build_page_statement() { + let _ = manifest_page_sql(&BlobReferenceRegistry::new()); + } + + /// Golden test — the repair statement is assembled from the same + /// registry as `manifest_page_sql`, so pin it byte-for-byte too. + /// If the registry ever changes what it produces at + /// `RefLevel::Manifest`, BOTH this test and + /// `manifest_page_statement_is_stable` above break together — an + /// operator using `?repair=true` shouldn't see the detection + /// formula report drift the repair formula can't clear. + /// + /// Ships the three-term formula (`storage.files` + + /// `storage.content_derived_blobs` + `storage.file_attached_blobs`) + /// twice — once in SET, once in the `<>` guard. Both must stay + /// identical so the guard is meaningful (else the UPDATE would fire + /// on drift the SET doesn't fix). + #[tokio::test] + async fn manifest_repair_statement_is_stable() { + let sql = manifest_repair_sql(&default_registry()); + let expected = r#"UPDATE storage.chunk_manifests m + SET ref_count = ((SELECT COUNT(*) FROM storage.files cnt_f + WHERE cnt_f.blob_hash = m.file_hash) + + (SELECT COUNT(*) FROM storage.content_derived_blobs cnt_d WHERE cnt_d.blob_hash = m.file_hash) + + (SELECT COUNT(*) FROM storage.file_attached_blobs cnt_a WHERE cnt_a.blob_hash = m.file_hash))::bigint + WHERE m.file_hash = $1 + AND m.ref_count <> ((SELECT COUNT(*) FROM storage.files cnt_f + WHERE cnt_f.blob_hash = m.file_hash) + + (SELECT COUNT(*) FROM storage.content_derived_blobs cnt_d WHERE cnt_d.blob_hash = m.file_hash) + + (SELECT COUNT(*) FROM storage.file_attached_blobs cnt_a WHERE cnt_a.blob_hash = m.file_hash))::bigint"#; + assert_eq!(sql, expected, "manifest repair statement changed:\n{sql}"); + } +} diff --git a/src/infrastructure/services/mod.rs b/src/infrastructure/services/mod.rs index b86e7560..b872a0bb 100644 --- a/src/infrastructure/services/mod.rs +++ b/src/infrastructure/services/mod.rs @@ -3,6 +3,7 @@ pub mod azure_blob_backend; pub mod backend_consistency_service; pub mod backend_migration_service; pub mod backend_rotate_service; +pub mod blob_diagnostics; pub mod blobs_consistency_service; pub mod cached_blob_backend; pub mod chunked_upload_service; @@ -27,9 +28,11 @@ pub mod folders_consistency_service; pub mod grant_cleanup_service; pub mod image_transcode_service; pub mod jwt_service; +pub mod last_seen_tracker; pub mod local_blob_backend; pub mod local_fs_mount_provider; pub mod login_lockout_service; +pub mod manifests_consistency_service; pub mod media_metadata_service; pub mod mock_email_sender; pub mod mount_provider_factory; @@ -49,14 +52,19 @@ pub mod plugins; pub mod recent_recording_hook; pub mod retry_blob_backend; pub mod s3_blob_backend; +pub mod satellites_consistency_service; pub mod search_index; pub mod session_cleanup_service; +pub mod session_liveness_gauges; pub mod share_unlock_cookie; pub mod smtp_email_sender; pub mod swappable_blob_backend; +pub mod thumb_attached_import_service; +pub mod thumb_derived_import_service; pub mod thumbnail_service; #[cfg(test)] mod thumbnail_service_test; +pub mod transcode_import_service; pub mod trash_cleanup_service; pub mod tree_etag_flush_service; pub mod webdav_dead_property_store; diff --git a/src/infrastructure/services/s3_blob_backend.rs b/src/infrastructure/services/s3_blob_backend.rs index c58ef2ce..6f877f2c 100644 --- a/src/infrastructure/services/s3_blob_backend.rs +++ b/src/infrastructure/services/s3_blob_backend.rs @@ -65,6 +65,32 @@ impl S3BlobBackend { let prefix = &hash[0..2]; format!("{}/{}.blob", prefix, hash) } + + /// Inverse of [`Self::object_key`] — the hash a key names, or `None` + /// when the key is not one we wrote. + /// + /// Deliberately strict, and paired with `object_key` so the round-trip + /// stays honest. Enumeration passes no prefix to S3, so this filter is + /// the *only* thing separating our namespace from everything else in + /// the bucket; a lenient match would feed a non-hash into + /// `object_key`, which slices `[0..2]` and would produce a nonsense + /// resume position. + fn hash_from_object_key(key: &str) -> Option { + let (prefix, rest) = key.split_once('/')?; + if prefix.len() != 2 || !prefix.chars().all(|c| c.is_ascii_hexdigit()) { + return None; + } + let stem = rest.strip_suffix(".blob")?; + if stem.len() != 64 || !stem.chars().all(|c| c.is_ascii_hexdigit()) { + return None; + } + // The shard must be the hash's own first two characters, or + // `object_key(hash)` would not reproduce this key. + if !stem.starts_with(prefix) { + return None; + } + Some(stem.to_string()) + } } impl BlobStorageBackend for S3BlobBackend { @@ -464,14 +490,21 @@ impl BlobStorageBackend for S3BlobBackend { None // Remote backend — no local path } - /// Enumerate blobs via S3 `ListObjectsV2`. Cursor is the S3 - /// continuation token verbatim (opaque). Filter: keys must - /// match `/<64-hex>.blob` — matches how `blob_key` writes - /// them — so any future non-blob namespace living in the same - /// bucket (e.g. `thumbnails/.jpg`) is skipped - /// automatically. No prefix passed to S3 so we get everything - /// in one paginated scan; the client-side filter enforces - /// correctness. + /// Enumerate blobs via S3 `ListObjectsV2`, in ascending hash order. + /// + /// The cursor is a **hash**, per the port contract — resumed via + /// `StartAfter`, not a continuation token. That is what lets a caller + /// resume the backend side of a merge-join from a checkpoint it + /// already holds; a continuation token would force re-enumeration + /// from the start on every resume. + /// + /// No prefix is passed to S3, so the scan covers the whole bucket and + /// [`Self::hash_from_object_key`] does the filtering. Keys that are + /// not ours come back as `unknowns` rather than being dropped, so an + /// operator can see what is sharing the bucket. **On a bucket shared + /// with other workloads that means every foreign object is reported + /// as an unknown on every sweep** — give OxiCloud its own bucket, or + /// expect the noise. fn list_blob_hashes( &self, cursor: Option, @@ -492,63 +525,110 @@ impl BlobStorageBackend for S3BlobBackend { }; Box::pin(async move { - let mut req = self - .client - .list_objects_v2() - .bucket(&self.bucket) - .max_keys(limit.min(1000) as i32); - if let Some(c) = cursor { - req = req.continuation_token(c); - } + // A page's cursor can only be the last blob hash on it, because + // the contract says the cursor IS a hash and `StartAfter` needs + // `object_key()` applied to it. A page holding only foreign keys + // therefore yields no cursor — and returning `None` there would + // end enumeration while the bucket still has objects, making an + // audit job under-report. That is the worst failure shape for a + // check whose entire purpose is finding missing data. + // + // So keep listing until the accumulated page holds at least one + // blob, or the bucket is exhausted. The continuation token is + // used only INSIDE this call and never escapes as a cursor. + // Bounded on foreign keys accumulated rather than on requests + // made: the request count scales with the caller's `limit`, so a + // request cap would fire on a healthy bucket merely because the + // caller paged finely. + const MAX_UNKNOWNS: usize = 10_000; - let resp = req.send().await.map_err(|e| { - DomainError::new( - ErrorKind::InternalError, - "Blob", - format!("S3 ListObjectsV2 failed: {e}"), - ) - })?; - - let objects = resp.contents.unwrap_or_default(); - let mut blobs: Vec = Vec::with_capacity(objects.len()); + let mut blobs: Vec = Vec::new(); let mut unknowns: Vec = Vec::new(); + let mut continuation: Option = None; + let mut requests = 0usize; + // Assigned on every path through the loop body before any exit. + let mut truncated; - for obj in objects { - let Some(key) = obj.key else { continue }; - let mtime = obj.last_modified.and_then(|ts| { - let secs = ts.secs(); - let nsecs = ts.subsec_nanos(); - chrono::DateTime::::from_timestamp(secs, nsecs) - }); + loop { + let mut req = self + .client + .list_objects_v2() + .bucket(&self.bucket) + .max_keys(limit.min(1000) as i32); + match (&continuation, &cursor) { + // Mid-loop: continue exactly where the last inner + // request stopped. + (Some(token), _) => req = req.continuation_token(token), + // First request: resume after the caller's hash. + (None, Some(c)) => req = req.start_after(Self::object_key(c)), + (None, None) => {} + } - // Canonical S3 key shape: `/<64-hex>.blob`. - // Anything else is a sidecar or foreign namespace - // (e.g. future `thumbnails/.jpg` if Ed adds - // that) — surface as an unknown so operators know - // it's there. Recovery framework can decide per- - // pattern how to act. - let is_canonical = key.split_once('/').and_then(|(prefix, rest)| { - if prefix.len() != 2 || !prefix.chars().all(|c| c.is_ascii_hexdigit()) { - return None; + let resp = req.send().await.map_err(|e| { + DomainError::new( + ErrorKind::InternalError, + "Blob", + format!("S3 ListObjectsV2 failed: {e}"), + ) + })?; + + requests += 1; + truncated = resp.is_truncated.unwrap_or(false); + continuation = resp.next_continuation_token; + + for obj in resp.contents.unwrap_or_default() { + let Some(key) = obj.key else { continue }; + let mtime = obj.last_modified.and_then(|ts| { + chrono::DateTime::::from_timestamp( + ts.secs(), + ts.subsec_nanos(), + ) + }); + + match Self::hash_from_object_key(&key) { + Some(hash) => blobs.push(BackendBlobEntry { hash, mtime }), + // Not ours: a spool file, a sidecar, or another + // workload sharing the bucket. Surfaced rather than + // dropped so operators can see it; the recovery + // framework decides per pattern how to act. + None => unknowns.push(BackendUnknownEntry { path: key, mtime }), } - rest.strip_suffix(".blob") - .filter(|stem| { - stem.len() == 64 && stem.chars().all(|c| c.is_ascii_hexdigit()) - }) - .map(|s| s.to_string()) - }); + } - match is_canonical { - Some(hash) => blobs.push(BackendBlobEntry { hash, mtime }), - None => unknowns.push(BackendUnknownEntry { path: key, mtime }), + if !blobs.is_empty() || !truncated { + break; + } + + // `is_truncated` with no token is a protocol violation, and a + // huge run of foreign keys means we would buffer the bucket to + // find one blob. Neither can produce a valid cursor, so fail + // loudly: a visible job failure beats a sweep that silently + // reports "no missing blobs" having read a fraction of them. + if continuation.is_none() || unknowns.len() >= MAX_UNKNOWNS { + return Err(DomainError::new( + ErrorKind::InternalError, + "Blob", + format!( + "S3 enumeration stalled after {requests} request(s) and {} \ + non-blob key(s) without reaching a blob, so no resume cursor \ + can be produced. Bucket '{}' likely holds a large foreign \ + namespace — give OxiCloud a dedicated bucket.", + unknowns.len(), + self.bucket, + ), + )); } } - let next_cursor = if resp.is_truncated.unwrap_or(false) { - resp.next_continuation_token + // Always a real hash: the loop above only exits with an empty + // `blobs` when the listing is exhausted, and then there is + // nothing to resume from. + let next_cursor = if truncated { + blobs.last().map(|entry| entry.hash.clone()) } else { None }; + Ok(BlobListPage { blobs, unknowns, @@ -623,3 +703,46 @@ where _ => format!("unknown SDK error: {err:?}"), } } + +#[cfg(test)] +mod tests { + use super::*; + + const H: &str = "0a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9"; + + /// The enumeration cursor is fed straight back into `object_key`, so a + /// key that does not round-trip would resume at the wrong position. + #[test] + fn object_key_round_trips_through_hash_from_object_key() { + let key = S3BlobBackend::object_key(H); + assert_eq!(key, format!("0a/{H}.blob")); + assert_eq!( + S3BlobBackend::hash_from_object_key(&key).as_deref(), + Some(H) + ); + } + + /// Each of these previously risked being treated as a hash and sliced + /// `[0..2]` to build a resume position. + #[test] + fn non_canonical_keys_are_rejected() { + let cases = [ + "0a/junk.tmp".to_string(), // spool file + "junk.tmp".to_string(), // no shard + "0a/junk".to_string(), // no suffix + "thumbnails/abc.jpg".to_string(), // foreign namespace + format!("0a/{H}.blob.corrupt"), // sidecar + format!("0a/{H}"), // suffix missing + format!("zz/{H}.blob"), // non-hex shard + format!("ff/{H}.blob"), // shard != hash prefix + format!("0a/{}.blob", &H[..63]), // wrong length + ]; + for key in &cases { + assert_eq!( + S3BlobBackend::hash_from_object_key(key), + None, + "must not be read as a blob: {key}" + ); + } + } +} diff --git a/src/infrastructure/services/satellites_consistency_service.rs b/src/infrastructure/services/satellites_consistency_service.rs new file mode 100644 index 00000000..99259263 --- /dev/null +++ b/src/infrastructure/services/satellites_consistency_service.rs @@ -0,0 +1,527 @@ +//! `satellites_consistency` — the last unbuilt row of the coverage matrix. +//! +//! Walks both satellite tables and reports mappings pointing at Blobs that no +//! longer exist. One job rather than two, because the tables are one concept +//! — the content-keyed and file-keyed halves of "things attached to a Blob" — +//! and the vocabulary already exists in `storage.copy_file_satellites`. +//! +//! ### Why nothing else finds these +//! +//! Every other job reasons from a Blob outwards: `blobs_consistency` and +//! `manifests_consistency` recompute refcounts for rows that exist, +//! `backend_consistency` merge-joins the registry against the backend. A +//! satellite row whose SOURCE is gone breaks none of those invariants — the +//! row holds a valid reference to a real artifact, the refcount is exactly +//! right, and the bytes are present on the backend. Every check agrees the +//! system is healthy. +//! +//! It is only wrong one level up: nothing will ever reap that source again, +//! so `purge_derived_blobs` can never fire, so the mapping is unreachable and +//! its artifact is pinned forever. A leak that looks like correctness, which +//! is why it survived four full suite runs before being named. +//! +//! That is not hypothetical — it shipped. Background thumbnail generation is +//! spawned and unawaited, so an upload deleted promptly had its render +//! complete after GC reaped the blob and then record three mappings to a +//! corpse. Fixed at the write side in `store_derived_blob`, which now refuses +//! a mapping whose source is gone; this job finds the ones already on disk, +//! which that fix cannot reach. +//! +//! ### Per-row checks +//! +//! * `derived_orphan_mapping` (`inconsistent`) — a `content_derived_blobs` +//! row whose `source_hash` has neither a manifest nor a blob row. Storage +//! that grows and never reclaims. +//! * `derived_dangling_blob` (`data_loss`) — its `blob_hash` has no Blob. +//! The mapping promises an artifact that is gone, so a read finds the row +//! and then fails. Recoverable in practice: a derived artifact is a pure +//! function of its source, so re-rendering restores it. +//! * `attached_dangling_blob` (`data_loss`) — the same for +//! `file_attached_blobs`, and **the one that cannot be recovered**. These +//! bytes are user-supplied — a client-generated PDF preview has no +//! server-side render path — so there is nothing to regenerate from. Same +//! finding shape as the derived case, materially higher stakes. +//! +//! There is deliberately no orphan-mapping check for the attached table: +//! `file_id` is `REFERENCES storage.files(id) ON DELETE CASCADE`, so a row +//! cannot outlive its file. The database enforces what the derived table +//! cannot, since a content hash has no row to point a foreign key at — which +//! is precisely why only that half could rot. +//! +//! Read-only, per the house default. Findings name a row rather than a range, +//! so recovery can act on them individually. + +use std::sync::Arc; + +use async_trait::async_trait; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::infrastructure::scheduler::{ + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome, + RunStatus, record_or_log, +}; + +pub const SATELLITES_CONSISTENCY_JOB_NAME: &str = "satellites_consistency"; + +/// Rows per page. Existence probes fold into the page query, so a page costs +/// one round-trip rather than `2 × rows`. +const BATCH_SIZE: i64 = 500; + +/// "Does this hash name a Blob?" — either table, because a Blob is a manifest +/// for CDC content and a bare `storage.blobs` row for legacy whole-file +/// content. Checking one would report every legacy blob as missing. +macro_rules! blob_exists { + ($col:literal) => { + concat!( + "(EXISTS (SELECT 1 FROM storage.chunk_manifests m WHERE m.file_hash = ", + $col, + ") OR EXISTS (SELECT 1 FROM storage.blobs b WHERE b.hash = ", + $col, + "))" + ) + }; +} + +pub struct SatellitesConsistencyCheck { + pool: Arc, +} + +#[derive(Debug, sqlx::FromRow)] +struct DerivedRow { + source_hash: String, + kind: String, + variant: String, + /// `None` on a NEGATIVE row — the derivation was attempted and is + /// known not to be worth storing for this content (a transcode that + /// came out larger, an undecodable source). Those rows point at + /// nothing on purpose and must not be read as dangling. + blob_hash: Option, + source_exists: bool, + artifact_exists: bool, +} + +#[derive(Debug, sqlx::FromRow)] +struct AttachedRow { + file_id: Uuid, + kind: String, + variant: String, + blob_hash: String, + uploaded_by: Uuid, + artifact_exists: bool, +} + +impl SatellitesConsistencyCheck { + pub fn new(pool: Arc) -> Self { + Self { pool } + } + + pub async fn register_recoverable_job( + self: Arc, + registry: &JobRegistry, + provider: &Arc, + ) -> Arc { + registry + .register_recoverable_job(self.clone(), provider.clone(), None) + .await; + self + } + + /// Both page queries key on the full primary key with a row-value + /// comparison, not on the first column: a source (or file) has several + /// variants, so a page boundary can fall inside one and advancing by the + /// first column alone would skip the rest. The tuple form also matches + /// the primary key's own ordering, so it stays index-friendly. + const DERIVED_PAGE_SQL: &'static str = concat!( + "SELECT d.source_hash, d.kind, d.variant, d.blob_hash, ", + blob_exists!("d.source_hash"), + " AS source_exists, ", + // A NEGATIVE row (NULL blob_hash) has no artifact BY DESIGN, so it + // counts as satisfied. Without this it reads as dangling: SQL + // comparison against NULL is NULL, so `EXISTS` is false, and every + // "this content is not worth transcoding" verdict would be reported + // as `data_loss`. The check has to be here rather than in the Rust + // arm below, so the column means "this row is in the state it + // should be" for both row shapes. + "(d.blob_hash IS NULL OR ", + blob_exists!("d.blob_hash"), + ") AS artifact_exists + FROM storage.content_derived_blobs d + WHERE ($1::text IS NULL + OR (d.source_hash, d.kind, d.variant) > ($1::text, $2::text, $3::text)) + ORDER BY d.source_hash, d.kind, d.variant + LIMIT $4" + ); + + const ATTACHED_PAGE_SQL: &'static str = concat!( + "SELECT a.file_id, a.kind, a.variant, a.blob_hash, a.uploaded_by, ", + blob_exists!("a.blob_hash"), + " AS artifact_exists + FROM storage.file_attached_blobs a + WHERE ($1::uuid IS NULL + OR (a.file_id, a.kind, a.variant) > ($1::uuid, $2::text, $3::text)) + ORDER BY a.file_id, a.kind, a.variant + LIMIT $4" + ); +} + +/// Cursor is `{phase}\n{a}\n{b}\n{c}`. +/// +/// The phase is what lets one job walk two tables and still resume exactly: +/// without it, a cursor from the attached pass would be replayed against the +/// derived table and silently re-scan or skip. Newline is a safe delimiter — +/// hashes are hex, uuids are uuids, `kind` comes from a CHECK constraint, and +/// `variant` is a size/format token. +#[derive(Debug, PartialEq, Clone, Copy)] +enum Phase { + Derived, + Attached, +} + +impl Phase { + fn as_str(self) -> &'static str { + match self { + Phase::Derived => "derived", + Phase::Attached => "attached", + } + } +} + +fn encode_cursor(phase: Phase, a: &str, b: &str, c: &str) -> Vec { + format!("{}\n{a}\n{b}\n{c}", phase.as_str()).into_bytes() +} + +type Cursor = Option<(Phase, String, String, String)>; + +fn decode_cursor(bytes: Vec) -> Result { + if bytes.is_empty() { + return Ok(None); + } + let s = String::from_utf8(bytes).map_err(|e| format!("not valid UTF-8: {e}"))?; + let mut parts = s.splitn(4, '\n'); + match (parts.next(), parts.next(), parts.next(), parts.next()) { + (Some("derived"), Some(a), Some(b), Some(c)) => { + Ok(Some((Phase::Derived, a.into(), b.into(), c.into()))) + } + (Some("attached"), Some(a), Some(b), Some(c)) => { + Ok(Some((Phase::Attached, a.into(), b.into(), c.into()))) + } + _ => Err(format!("malformed cursor: {s:?}")), + } +} + +#[async_trait] +impl RecoverableJobHandler for SatellitesConsistencyCheck { + fn name(&self) -> &str { + SATELLITES_CONSISTENCY_JOB_NAME + } + + fn description(&self) -> &'static str { + "Walks both satellite tables — content_derived_blobs (thumbnails \ + keyed by source content) and file_attached_blobs (previews keyed \ + by file) — and reports mappings whose source or target no longer \ + exists. Nothing else finds these: every other job reasons from a \ + Blob outwards, and a satellite row pointing at a deleted source \ + breaks none of their invariants. Read-only." + } + + async fn count_total(&self) -> Option { + sqlx::query_as::<_, (i64,)>( + "SELECT (SELECT COUNT(*) FROM storage.content_derived_blobs) + + (SELECT COUNT(*) FROM storage.file_attached_blobs)", + ) + .fetch_one(self.pool.as_ref()) + .await + .ok() + .map(|(n,)| n.max(0) as u64) + } + + async fn run_resumable( + &self, + store: &dyn JobStore, + _args: &JobRunArgs, + resume_cursor: Option>, + ) -> RunOutcome { + let start = match resume_cursor.map(decode_cursor).transpose() { + Ok(c) => c.flatten(), + Err(message) => return RunOutcome::Failed { message }, + }; + + let mut finding_count = 0u64; + + // ── Phase 1: content-keyed ─────────────────────────────────────── + // Skipped entirely when resuming mid-attached, since that phase runs + // strictly after this one. + let mut derived_cursor = match &start { + Some((Phase::Attached, ..)) => None, + Some((Phase::Derived, a, b, c)) => Some((a.clone(), b.clone(), c.clone())), + None => None, + }; + let skip_derived = matches!(&start, Some((Phase::Attached, ..))); + + if !skip_derived { + loop { + if let Some(outcome) = poll_cancel( + store, + derived_cursor + .as_ref() + .map(|(a, b, c)| encode_cursor(Phase::Derived, a, b, c)), + ) + .await + { + return outcome; + } + + let (ch, ck, cv) = match &derived_cursor { + Some((a, b, c)) => (Some(a.as_str()), Some(b.as_str()), Some(c.as_str())), + None => (None, None, None), + }; + + let rows: Vec = match sqlx::query_as(Self::DERIVED_PAGE_SQL) + .bind(ch) + .bind(ck) + .bind(cv) + .bind(BATCH_SIZE) + .fetch_all(self.pool.as_ref()) + .await + { + Ok(r) => r, + Err(e) => { + return RunOutcome::Failed { + message: format!("derived page: {e}"), + }; + } + }; + if rows.is_empty() { + break; + } + + for row in &rows { + if !row.source_exists { + finding_count += 1; + record_or_log( + store, + SATELLITES_CONSISTENCY_JOB_NAME, + "derived_orphan_mapping", + "inconsistent", + None, + serde_json::json!({ + "source_hash": row.source_hash, + "kind": row.kind, + "variant": row.variant, + "blob_hash": row.blob_hash, + "note": "source Blob is gone, so purge_derived_blobs can never \ + fire; this row pins its artifact forever", + }), + ) + .await; + } + if !row.artifact_exists { + finding_count += 1; + record_or_log( + store, + SATELLITES_CONSISTENCY_JOB_NAME, + "derived_dangling_blob", + "data_loss", + None, + serde_json::json!({ + "source_hash": row.source_hash, + "kind": row.kind, + "variant": row.variant, + "blob_hash": row.blob_hash, + "recoverable": true, + "note": "artifact missing; derived content is a pure function of \ + its source, so re-rendering restores it", + }), + ) + .await; + } + } + + let scanned = rows.len() as u64; + let last = rows.last().unwrap(); + derived_cursor = Some(( + last.source_hash.clone(), + last.kind.clone(), + last.variant.clone(), + )); + if let Err(e) = store + .checkpoint( + encode_cursor(Phase::Derived, &last.source_hash, &last.kind, &last.variant), + scanned, + ) + .await + { + return RunOutcome::Failed { + message: format!("checkpoint: {e}"), + }; + } + if scanned < BATCH_SIZE as u64 { + break; + } + } + } + + // ── Phase 2: file-keyed ────────────────────────────────────────── + // No orphan-mapping check here: `file_id` is ON DELETE CASCADE, so a + // row cannot outlive its file. Only the artifact side can rot. + let mut attached_cursor: Option<(Uuid, String, String)> = match &start { + Some((Phase::Attached, a, b, c)) => match Uuid::parse_str(a) { + Ok(id) => Some((id, b.clone(), c.clone())), + Err(e) => { + return RunOutcome::Failed { + message: format!("attached cursor is not a uuid: {e}"), + }; + } + }, + _ => None, + }; + + loop { + if let Some(outcome) = poll_cancel( + store, + attached_cursor + .as_ref() + .map(|(a, b, c)| encode_cursor(Phase::Attached, &a.to_string(), b, c)), + ) + .await + { + return outcome; + } + + let (ch, ck, cv) = match &attached_cursor { + Some((a, b, c)) => (Some(*a), Some(b.as_str()), Some(c.as_str())), + None => (None, None, None), + }; + + let rows: Vec = match sqlx::query_as(Self::ATTACHED_PAGE_SQL) + .bind(ch) + .bind(ck) + .bind(cv) + .bind(BATCH_SIZE) + .fetch_all(self.pool.as_ref()) + .await + { + Ok(r) => r, + Err(e) => { + return RunOutcome::Failed { + message: format!("attached page: {e}"), + }; + } + }; + if rows.is_empty() { + break; + } + + for row in &rows { + if !row.artifact_exists { + finding_count += 1; + record_or_log( + store, + SATELLITES_CONSISTENCY_JOB_NAME, + "attached_dangling_blob", + "data_loss", + None, + serde_json::json!({ + "file_id": row.file_id, + "kind": row.kind, + "variant": row.variant, + "blob_hash": row.blob_hash, + "uploaded_by": row.uploaded_by, + "recoverable": false, + "note": "UNRECOVERABLE: these bytes were user-supplied and have no \ + server-side render path, so nothing can regenerate them", + }), + ) + .await; + } + } + + let scanned = rows.len() as u64; + let last = rows.last().unwrap(); + attached_cursor = Some((last.file_id, last.kind.clone(), last.variant.clone())); + if let Err(e) = store + .checkpoint( + encode_cursor( + Phase::Attached, + &last.file_id.to_string(), + &last.kind, + &last.variant, + ), + scanned, + ) + .await + { + return RunOutcome::Failed { + message: format!("checkpoint: {e}"), + }; + } + if scanned < BATCH_SIZE as u64 { + break; + } + } + + tracing::info!( + target: "oxicloud::consistency", + event = "satellites_consistency.completed", + run_id = %store.run_id(), + finding_count = finding_count, + "satellites_consistency completed with {} finding(s)", + finding_count + ); + + RunOutcome::completed() + } +} + +/// Cooperative cancel, shared by both phases so neither can forget it. +async fn poll_cancel(store: &dyn JobStore, cursor: Option>) -> Option { + match store.status().await { + Ok(RunStatus::CancelRequested) => Some(RunOutcome::Paused { + cursor: cursor.unwrap_or_default(), + }), + Ok(_) => None, + Err(e) => Some(RunOutcome::Failed { + message: format!("status poll: {e}"), + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The phase is what lets one job walk two tables and resume exactly. + /// Without it an attached cursor would be replayed against the derived + /// table, silently re-scanning or skipping — an audit job under-reporting + /// is the worst failure available to it. + #[test] + fn cursor_round_trips_and_keeps_its_phase() { + for phase in [Phase::Derived, Phase::Attached] { + let encoded = encode_cursor(phase, "0a1b", "thumbnail", "preview.webp"); + assert_eq!( + decode_cursor(encoded).unwrap(), + Some(( + phase, + "0a1b".to_string(), + "thumbnail".to_string(), + "preview.webp".to_string() + )) + ); + } + } + + #[test] + fn empty_cursor_starts_from_the_beginning() { + assert_eq!(decode_cursor(Vec::new()).unwrap(), None); + } + + /// Loudly, rather than silently restarting: a corrupt checkpoint that + /// reads as "start over" gives a job that never finishes and never says + /// why. + #[test] + fn malformed_cursor_is_an_error() { + assert!(decode_cursor(b"only-one-field".to_vec()).is_err()); + assert!(decode_cursor(b"bogus\na\nb\nc".to_vec()).is_err()); + } +} diff --git a/src/infrastructure/services/session_cleanup_service.rs b/src/infrastructure/services/session_cleanup_service.rs index c909540a..9a19d655 100644 --- a/src/infrastructure/services/session_cleanup_service.rs +++ b/src/infrastructure/services/session_cleanup_service.rs @@ -35,7 +35,7 @@ use tracing::{error, info}; use crate::domain::repositories::session_repository::SessionRepository; use crate::infrastructure::repositories::SessionPgRepository; -use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs}; +use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs, Mutates}; /// How long a session row survives past its `expires_at` before this /// janitor deletes it. Enough time for a security review of a @@ -84,6 +84,17 @@ impl JobHandler for SessionCleanupService { Self::JOB_NAME } + fn description(&self) -> &'static str { + "Deletes session rows long past their expiry. They cannot \ + authenticate — expiry is checked at every auth path — but the row \ + keeps a forensic trail (which user, from which IP, minted how) \ + for a retention window after expiry, then becomes dead weight." + } + + fn mutates(&self) -> Mutates { + Mutates::Always + } + /// Runs one bulk-delete of long-expired session rows. `count` on /// the returned `JobOutcome::Ok` is the number of rows dropped /// this tick; `extra` records the retention window operators can diff --git a/src/infrastructure/services/session_liveness_gauges.rs b/src/infrastructure/services/session_liveness_gauges.rs new file mode 100644 index 00000000..cc6f0cfe --- /dev/null +++ b/src/infrastructure/services/session_liveness_gauges.rs @@ -0,0 +1,169 @@ +//! Prometheus session-liveness gauges — periodic polling of +//! `auth.sessions` to publish three gauges the `/metrics` scraper +//! reads: +//! +//! - `oxicloud_sessions_online` — non-revoked rows observed in the +//! last [`ONLINE_WINDOW`](crate::application::dtos::session_dto::ONLINE_WINDOW). +//! **Per-session count**, not per-user — one user with three +//! devices contributes three. +//! - `oxicloud_sessions_online_users` — DISTINCT `user_id` behind +//! those online sessions. The multi-device factor is exactly +//! `sessions_online / sessions_online_users`. +//! - `oxicloud_sessions_total_non_revoked` — long-tail total, +//! including mobile clients still holding a refresh token they +//! haven't used in weeks. Useful sanity signal on the dashboard. +//! +//! **Naming — "online" vs "active".** The word "active" is already +//! spoken for by the session *lifecycle* (Active | Expired | +//! Revoked in the admin panel). Presence (recently-seen) is +//! orthogonal and uses "online" throughout the UI, DTO +//! (`SessionSummaryDto::is_online`), and these gauges — so a +//! dashboard graph and a per-row green-dot badge have the same +//! label root. Terminology decided 2026-08-18; see +//! `docs/plan/sessions.md`. +//! +//! **Cadence.** Poller ticks every [`POLL_INTERVAL`] (30 s). Three +//! `COUNT(*)` reads on the maintenance pool per tick — negligible +//! load on tens-of-thousands-of-rows tables thanks to the partial +//! index `idx_sessions_last_seen_at` (partial on `revoked = FALSE`, +//! which every query below filters on). +//! +//! **When it runs.** Spawned from DI only when auth is enabled AND +//! `OXICLOUD_METRICS_LISTEN` is set (recorder installed). Without +//! the recorder, `metrics::gauge!(...)` is a no-op — spawning +//! anyway would still hit PG every 30 s for values nobody reads. +//! +//! See `docs/plan/sessions.md` for the full design. + +use std::sync::Arc; +use std::time::Duration; + +use sqlx::PgPool; + +use crate::application::dtos::session_dto::ONLINE_WINDOW; + +/// Poll cadence. Matches the [`LastSeenTracker`](super::last_seen_tracker) +/// flush cadence so the gauges converge one tick after the tracker +/// flushes — no need to sync the two. +const POLL_INTERVAL: Duration = Duration::from_secs(30); + +/// Spawn the session-liveness poller. Detached — the task lives +/// for the runtime's lifetime; there's no mid-process reason to +/// stop reporting gauges. +/// +/// Emits an initial poll on spawn so the very first `/metrics` +/// scrape after boot returns real values instead of the recorder's +/// zero-initialised default. +pub fn spawn(maintenance_pool: Arc) { + tokio::spawn(async move { + // Immediate first tick — a scraper hitting `/metrics` in + // the first 30 s otherwise sees `oxicloud_sessions_online + // 0` even on a busy server. Warmup query is cheap. + if let Err(err) = poll_once(&maintenance_pool).await { + tracing::warn!( + target: "oxicloud::sessions", + error = %err, + "initial session-liveness poll failed", + ); + } + + let mut ticker = tokio::time::interval(POLL_INTERVAL); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // Consume the first tick — `interval` fires immediately on + // creation and we've already done the warmup above. + ticker.tick().await; + + loop { + ticker.tick().await; + if let Err(err) = poll_once(&maintenance_pool).await { + tracing::warn!( + target: "oxicloud::sessions", + error = %err, + "session-liveness poll failed; keeping last-known gauge values", + ); + } + } + }); + tracing::info!( + target: "oxicloud::sessions", + poll_interval_secs = POLL_INTERVAL.as_secs(), + online_window_secs = ONLINE_WINDOW.as_secs(), + "📊 session-liveness gauges spawned", + ); +} + +/// One poll cycle. Three lightweight `COUNT` reads → three gauge +/// updates. Errors propagate to the caller (loop logs + retries +/// next tick; gauges keep their last-known value in the interim, +/// which is the honest thing to publish — a temporary PG blip is +/// not a "sessions dropped to zero" event). +async fn poll_once(pool: &PgPool) -> Result<(), sqlx::Error> { + // NOTE: `ONLINE_WINDOW` is a Duration; PG expects the interval + // in seconds via `make_interval` (portable across sqlx driver + // versions). Casting once at bind time is cheaper than an + // `INTERVAL '$1 seconds'` string interp and keeps the query + // parameterised. + let online_secs: f64 = ONLINE_WINDOW.as_secs_f64(); + + let online: i64 = sqlx::query_scalar( + r#" + SELECT COUNT(*) FROM auth.sessions + WHERE revoked = FALSE + AND last_seen_at > NOW() - make_interval(secs => $1) + "#, + ) + .bind(online_secs) + .fetch_one(pool) + .await?; + + let online_users: i64 = sqlx::query_scalar( + r#" + SELECT COUNT(DISTINCT user_id) FROM auth.sessions + WHERE revoked = FALSE + AND last_seen_at > NOW() - make_interval(secs => $1) + "#, + ) + .bind(online_secs) + .fetch_one(pool) + .await?; + + let total_non_revoked: i64 = sqlx::query_scalar( + r#" + SELECT COUNT(*) FROM auth.sessions WHERE revoked = FALSE + "#, + ) + .fetch_one(pool) + .await?; + + // metrics-exporter-prometheus takes f64 gauges; the raw COUNT + // fits into f64 precisely up to 2^53, well past any realistic + // session-row count. `describe_gauge!` is called once at first + // emission and cached in the recorder — the second/third tick + // just updates the value. + metrics::describe_gauge!( + "oxicloud_sessions_online", + "Non-revoked sessions observed in the last ONLINE_WINDOW." + ); + metrics::gauge!("oxicloud_sessions_online").set(online as f64); + + metrics::describe_gauge!( + "oxicloud_sessions_online_users", + "Distinct users behind sessions observed in the last ONLINE_WINDOW." + ); + metrics::gauge!("oxicloud_sessions_online_users").set(online_users as f64); + + metrics::describe_gauge!( + "oxicloud_sessions_total_non_revoked", + "Total non-revoked sessions regardless of last-seen recency." + ); + metrics::gauge!("oxicloud_sessions_total_non_revoked").set(total_non_revoked as f64); + + tracing::debug!( + target: "oxicloud::sessions", + online, + online_users, + total_non_revoked, + "session-liveness gauges updated", + ); + Ok(()) +} diff --git a/src/infrastructure/services/thumb_attached_import_service.rs b/src/infrastructure/services/thumb_attached_import_service.rs new file mode 100644 index 00000000..92917d55 --- /dev/null +++ b/src/infrastructure/services/thumb_attached_import_service.rs @@ -0,0 +1,619 @@ +//! `thumb_attached_import` — backfill `storage.file_attached_blobs` from the +//! `ext-{file_id}.jpg` sidecars that predate it. +//! +//! Second half of step 10's migration, and the twin of +//! `thumb_derived_import`. These are the thumbnails a *user* supplied — the +//! SPA's client-side generator, notably for PDFs, which have no server-side +//! render path at all. They live only as +//! `{thumbnails_root}/{size}/ext-{file_id}.jpg` on local disk. +//! +//! Until a row exists, a **copy of the file loses the preview**: the sidecar +//! is keyed by `file_id`, no copy path duplicates it, and the server silently +//! falls back to rendering from the source (or to nothing, for a PDF). That +//! is the bug `file_attached_blobs` closed for new uploads; this job closes +//! it for everything already on disk. +//! +//! ### File-keyed, and that is the whole point +//! +//! These bytes are **not** derivable from the file's content, so they must +//! never be content-keyed. Sharing one user's uploaded preview across every +//! file with identical content is the poisoning vector the table split +//! exists to prevent — see `docs/plan/derived-blobs.md`. `thumb_derived_import` +//! deliberately rejects `ext-` names for the same reason, and the two jobs +//! are separate so neither can drift into the other's keying. +//! +//! ### Idempotence needs care here +//! +//! Unlike the derived twin, `store_attached_blob` is `ON CONFLICT DO UPDATE`: +//! calling it for a row that already exists releases the previous reference +//! and takes a new one. Harmless once, but a job that did it on every run +//! would churn refcounts. So each file is skipped when a row is already +//! present, and the store is only reached on a genuine insert. +//! +//! ### Multi-instance caveat +//! +//! Sidecars are local, so this migrates only the instance it runs on. Phase 3 +//! must be gated on every instance reporting an empty tail. + +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use sqlx::PgPool; +use tokio::fs; +use uuid::Uuid; + +use crate::application::ports::thumbnail_ports::ThumbnailSize; +use crate::infrastructure::scheduler::{ + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, Mutates, RecoverableJobHandler, + RunOutcome, RunStatus, record_or_log, +}; +use crate::infrastructure::services::dedup_service::DedupService; +// The readback-then-unlink rule is shared, not copied: two versions of it +// would be two chances to weaken one, and this is the check standing between +// a migration and permanent loss. +use crate::infrastructure::services::thumb_derived_import_service::ThumbDerivedImport; + +pub const THUMB_ATTACHED_IMPORT_JOB_NAME: &str = "thumb_attached_import"; + +/// Files handled between checkpoints — a read plus at most a blob write each. +const BATCH_SIZE: usize = 100; + +/// `uploaded_by` for imported rows. +/// +/// Disk records no uploader, and the column is deliberately `NOT NULL` with no +/// FK so provenance survives a user deletion. A sentinel says "imported, real +/// uploader unknown" honestly; inventing an owner — the file's `created_by`, +/// say — would fabricate provenance that could later be read as evidence an +/// Editor replaced someone's preview. +const IMPORTED_UPLOADER: Uuid = Uuid::nil(); + +pub struct ThumbAttachedImport { + thumbnails_root: PathBuf, + dedup: Arc, + pool: Arc, +} + +impl ThumbAttachedImport { + pub fn new(thumbnails_root: PathBuf, dedup: Arc, pool: Arc) -> Self { + Self { + thumbnails_root, + dedup, + pool, + } + } + + pub async fn register_recoverable_job( + self: Arc, + registry: &JobRegistry, + provider: &Arc, + ) -> Arc { + // On-demand, matching `thumb_derived_import` — the boot run in repair + // mode is the migration, and a tick could not finish it anyway + // because ticks never pass `repair`. See that job for the reasoning. + registry + .register_recoverable_job(self.clone(), provider.clone(), None) + .await; + self + } + + /// The file id an external sidecar names, or `None` when the file is not + /// one of ours. + /// + /// Requires a parseable UUID: the name is about to be used as a foreign + /// key, and a malformed one should be reported rather than fed to the + /// database. + fn file_id_from_sidecar_name(name: &str) -> Option { + let stem = name.strip_prefix("ext-")?.strip_suffix(".jpg")?; + Uuid::parse_str(stem).ok() + } + + /// Sorted external-sidecar filenames for one size directory. + /// + /// Sorted because the cursor resumes by skipping everything at or before + /// it, which only works over a stable order. + /// + /// Takes the root rather than reading `self`, so the walk — the half that + /// decides which files this job claims, and therefore which keying they + /// get — is testable against a temp directory with no database in sight. + async fn sidecar_names(root: &std::path::Path, size: ThumbnailSize) -> Vec { + let dir = root.join(size.dir_name()); + let Ok(mut entries) = fs::read_dir(&dir).await else { + return Vec::new(); + }; + let mut names = Vec::new(); + while let Ok(Some(entry)) = entries.next_entry().await { + if let Some(name) = entry.file_name().to_str() + && Self::file_id_from_sidecar_name(name).is_some() + { + names.push(name.to_string()); + } + } + names.sort(); + names + } + + /// Does the file still exist? Checked explicitly rather than letting the + /// foreign key reject the insert, so an orphaned sidecar is *counted* as + /// an orphan instead of surfacing as an opaque constraint error. + /// `SELECT EXISTS(...)`, deliberately, rather than `SELECT 1 … LIMIT 1`. + /// + /// PostgreSQL types a bare `1` as `int4`, so decoding it as `i64` fails — + /// and because a decode error is indistinguishable from "no row" once + /// swallowed, every sidecar would be misreported as an orphan and nothing + /// would import. `EXISTS` yields a real `bool` and always returns exactly + /// one row, so absence means absence. + /// + /// A query error still degrades to `false`, which is the safe direction: + /// the file is reported as an orphan and left on disk for the operator, + /// rather than imported against a row that may not exist. + async fn file_exists(&self, file_id: Uuid) -> bool { + sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM storage.files WHERE id = $1)") + .bind(file_id) + .fetch_one(self.pool.as_ref()) + .await + .unwrap_or(false) + } +} + +#[async_trait] +impl RecoverableJobHandler for ThumbAttachedImport { + fn name(&self) -> &str { + THUMB_ATTACHED_IMPORT_JOB_NAME + } + + fn description(&self) -> &'static str { + "Migrates USER-UPLOADED previews (ext-{file_id}.jpg) into \ + file-keyed blob storage. Until a row exists, copying a file loses \ + its preview: the sidecar is keyed by file id and no copy path \ + duplicates it. These bytes have no server-side render path, so \ + unlike rendered thumbnails they cannot be regenerated." + } + + fn mutates(&self) -> Mutates { + Mutates::Always + } + + fn repair_description(&self) -> Option<&'static str> { + Some( + "Also DELETES each sidecar once its replacement has been read \ + back. Previews whose file no longer exists are deleted without \ + a readback — nothing can reference them again. Irreversible, \ + and these bytes cannot be regenerated, so the readback is the \ + only safeguard.", + ) + } + + async fn count_total(&self) -> Option { + let mut total = 0u64; + for size in ThumbnailSize::all() { + total += Self::sidecar_names(&self.thumbnails_root, *size) + .await + .len() as u64; + } + Some(total) + } + + async fn run_resumable( + &self, + store: &dyn JobStore, + args: &JobRunArgs, + resume_cursor: Option>, + ) -> RunOutcome { + // Cursor is `{size_dir}/{filename}`, matching thumb_derived_import: + // sizes walk in `ThumbnailSize::all()` order and names are sorted + // within each, so the pair totally orders the traversal. + let cursor: Option = match resume_cursor { + None => None, + Some(b) if b.is_empty() => None, + Some(b) => match String::from_utf8(b) { + Ok(s) => Some(s), + Err(e) => { + return RunOutcome::Failed { + message: format!("invalid cursor: not valid UTF-8: {e}"), + }; + } + }, + }; + + let mut imported = 0u64; + let mut already = 0u64; + let mut orphaned = 0u64; + let mut deleted = 0u64; + let mut unverified = 0u64; + // Same opt-in as thumb_derived_import: `?repair=true`. + // + // The readback before unlinking matters more here than there. These + // sidecars are the ones that CANNOT be regenerated — a client-uploaded + // PDF preview has no server-side render path — so it is not + // belt-and-braces, it is the only thing between a migration and + // permanent loss. + let delete_imported = args.repair; + let mut failed = 0u64; + let mut since_checkpoint = 0usize; + + for size in ThumbnailSize::all() { + let dir_name = size.dir_name().to_string(); + for name in Self::sidecar_names(&self.thumbnails_root, *size).await { + let position = format!("{dir_name}/{name}"); + + if let Some(c) = &cursor + && position.as_str() <= c.as_str() + { + continue; + } + + match store.status().await { + Ok(RunStatus::CancelRequested) => { + return RunOutcome::Paused { + cursor: position.into_bytes(), + }; + } + Ok(_) => {} + Err(e) => { + return RunOutcome::Failed { + message: format!("status poll: {e}"), + }; + } + } + + let Some(file_id) = Self::file_id_from_sidecar_name(&name) else { + continue; + }; + let file_id_str = file_id.to_string(); + + // Already mapped. Checked BEFORE storing, because + // `store_attached_blob` is ON CONFLICT DO UPDATE and would + // release then retake the reference on every run. + if let Some(existing) = self + .dedup + .find_attached_blob(&file_id_str, "preview", &dir_name) + .await + { + already += 1; + // Drains on a later run too: importing first and enabling + // deletion afterwards is the expected operator sequence, + // so reaching here is the common path rather than an edge + // case. + if delete_imported { + let path = self.thumbnails_root.join(&dir_name).join(&name); + if ThumbDerivedImport::verify_and_unlink( + &self.dedup, + THUMB_ATTACHED_IMPORT_JOB_NAME, + &file_id_str, + &existing.blob_hash, + &path, + ) + .await + { + deleted += 1; + } else { + unverified += 1; + record_or_log( + store, + THUMB_ATTACHED_IMPORT_JOB_NAME, + "sidecar_delete_unverified", + "anomaly", + None, + serde_json::json!({ + "path": position, + "file_id": file_id_str, + "note": "attached blob did not read back; sidecar kept", + }), + ) + .await; + } + } + } else if !self.file_exists(file_id).await { + // The file is gone, so this sidecar is unimportable: the + // FK on `file_id` would reject the row. Mirrors the + // dead-source case in thumb_derived_import. + // + // Reported by default — a destructive default on a + // migration is what no-silent-auto-repair forbids — and + // deleted under `repair`, because otherwise it is + // rediscovered on every run, the tail never empties, and + // step 10e's gate never opens. + // + // Safe to delete despite these being the non-regenerable + // bytes: the preview is keyed to a `file_id` that no + // longer exists, so nothing can ever reference it again. + // Unrecoverable and unreachable are different things, and + // this is both. + // + // No readback before unlinking, unlike the imported path: + // there is no row and no blob to read back, and nothing to + // regenerate from either. + orphaned += 1; + let mut removed = false; + if delete_imported { + let path = self.thumbnails_root.join(&dir_name).join(&name); + if fs::remove_file(&path).await.is_ok() { + deleted += 1; + removed = true; + // Explicit: nothing to verify against, so this + // bypasses verify_and_unlink. Worth auditing + // loudest of all — these bytes were + // user-supplied and cannot be regenerated, even + // though the file that owned them is gone. + crate::infrastructure::services::thumb_derived_import_service::audit_sidecar_deleted( + THUMB_ATTACHED_IMPORT_JOB_NAME, + "orphaned", + &file_id_str, + "-", + &path, + ); + } + } + // Recorded in BOTH modes — see the twin in + // thumb_derived_import. Deleting a non-regenerable + // user-uploaded preview and reporting nothing is the + // worst version of this: the one outcome an operator + // needs in the run drawer was the one it withheld. + record_or_log( + store, + THUMB_ATTACHED_IMPORT_JOB_NAME, + "attached_sidecar_orphan", + // `anomaly` renders as "notices"; `detail.deleted` + // is what says whether the run acted. See the + // derived twin. + "anomaly", + None, + serde_json::json!({ + "path": position, + "file_id": file_id_str, + "deleted": removed, + "note": if removed { + "no storage.files row; sidecar was unimportable and has been \ + deleted — nothing can reference it again" + } else { + "no storage.files row; unimportable, and deleted on a repair \ + run since nothing can reference it again" + }, + }), + ) + .await; + } else { + let path = self.thumbnails_root.join(&dir_name).join(&name); + match fs::read(&path).await { + Ok(data) => { + match self + .dedup + .store_attached_blob( + &file_id_str, + "preview", + &dir_name, + // store_external_thumbnail re-encodes to + // JPEG before writing, so the extension + // is authoritative here. + "image/jpeg", + Bytes::from(data), + IMPORTED_UPLOADER, + ) + .await + { + Ok(attached_hash) => { + imported += 1; + if delete_imported { + if ThumbDerivedImport::verify_and_unlink( + &self.dedup, + THUMB_ATTACHED_IMPORT_JOB_NAME, + &file_id_str, + &attached_hash, + &path, + ) + .await + { + deleted += 1; + } else { + unverified += 1; + record_or_log( + store, + THUMB_ATTACHED_IMPORT_JOB_NAME, + "sidecar_delete_unverified", + "anomaly", + None, + serde_json::json!({ + "path": position, + "file_id": file_id_str, + "note": "attached blob did not read back; sidecar kept", + }), + ) + .await; + } + } + } + Err(e) => { + failed += 1; + record_or_log( + store, + THUMB_ATTACHED_IMPORT_JOB_NAME, + "attached_import_failed", + "anomaly", + None, + serde_json::json!({ + "path": position, + "file_id": file_id_str, + "error": format!("{e}"), + "note": "sidecar left in place; safe to re-run", + }), + ) + .await; + } + } + } + Err(e) => { + failed += 1; + record_or_log( + store, + THUMB_ATTACHED_IMPORT_JOB_NAME, + "attached_sidecar_unreadable", + "anomaly", + None, + serde_json::json!({ + "path": position, + "error": format!("{e}"), + }), + ) + .await; + } + } + } + + since_checkpoint += 1; + if since_checkpoint >= BATCH_SIZE { + if let Err(e) = store + .checkpoint(position.clone().into_bytes(), since_checkpoint as u64) + .await + { + return RunOutcome::Failed { + message: format!("checkpoint: {e}"), + }; + } + since_checkpoint = 0; + } + } + } + + // Flush the tail — see the derived twin. The loop only checkpoints + // on a full batch, so the remainder went uncounted: a 105-file run + // reported `scanned_count: 100`, and a run shorter than one batch + // reported zero and left the progress bar at zero throughout. + if since_checkpoint > 0 + && let Err(e) = store.checkpoint(Vec::new(), since_checkpoint as u64).await + { + return RunOutcome::Failed { + message: format!("final checkpoint: {e}"), + }; + } + + // Both jobs attempt the teardown, and it no-ops unless the tree is + // drained of files EITHER of them claims. Without this, whichever + // job runs last leaves an empty `.thumbnails/` behind until the + // next boot; with it, the tree disappears in the same run that + // empties it, whatever order the two ran in. + if delete_imported { + crate::infrastructure::services::thumb_derived_import_service::teardown_if_drained( + &self.thumbnails_root, + THUMB_ATTACHED_IMPORT_JOB_NAME, + &store.run_id().to_string(), + ) + .await; + } + + tracing::info!( + target: "oxicloud::dedup", + event = "thumb_attached_import.completed", + run_id = %store.run_id(), + imported = imported, + already_present = already, + orphaned = orphaned, + failed = failed, + deleted = deleted, + unverified = unverified, + "thumb_attached_import: {imported} imported, {already} already present, \ + {orphaned} orphaned, {failed} failed, {deleted} sidecar(s) deleted, \ + {unverified} kept unverified" + ); + + // Same reasoning as the derived twin: what the run did belongs on + // the run row, not only in the process log. + RunOutcome::completed_with(serde_json::json!({ + "imported": imported, + "already_present": already, + "deleted": deleted, + "unverified": unverified, + "orphaned": orphaned, + "failed": failed, + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const UUID: &str = "3f2b1c00-1111-2222-3333-444455556666"; + const HASH: &str = "0a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9"; + + #[test] + fn accepts_an_external_sidecar_name() { + assert_eq!( + ThumbAttachedImport::file_id_from_sidecar_name(&format!("ext-{UUID}.jpg")), + Some(Uuid::parse_str(UUID).unwrap()) + ); + } + + /// The other half of the partition. Reuses the same legacy tree as + /// `thumb_derived_import`'s test on purpose: the two jobs run over one + /// directory, so the property that matters is that together they claim + /// every real sidecar exactly once, and neither takes the other's. + #[tokio::test] + async fn walk_claims_only_uploaded_previews() { + use crate::infrastructure::services::thumb_derived_import_service::ThumbDerivedImport; + + let tmp = + crate::infrastructure::services::thumb_derived_import_service::tests::legacy_tree() + .await; + + let attached = ThumbAttachedImport::sidecar_names(tmp.path(), ThumbnailSize::Preview).await; + let derived = ThumbDerivedImport::sidecar_names(tmp.path(), ThumbnailSize::Preview).await; + + assert_eq!( + attached, + vec!["ext-3f2b1c00-1111-2222-3333-444455556666.jpg".to_string()], + "must claim the uploaded preview and nothing else" + ); + + // Disjoint: no file is imported under both keyings, which would take + // two references and — worse — content-key user-supplied bytes. + for a in &attached { + assert!( + !derived.contains(a), + "both jobs claimed {a}; keying would be ambiguous" + ); + } + // And nothing real is dropped: README.txt is the only unclaimed file. + // Two content-keyed .webp, one content-keyed .jpg, one ext- upload. + // The .jpg pair is the interesting one: same extension, opposite + // keying, and only the `ext-` prefix separates them. + assert_eq!( + attached.len() + derived.len(), + 4, + "every real sidecar must be claimed exactly once between the two jobs" + ); + } + + /// The content-keyed sidecars belong to `thumb_derived_import`. Importing + /// one here would file-key bytes that are shared across every file with + /// the same content, so each such file would take its own reference to + /// content it does not own. + #[test] + fn rejects_content_keyed_and_malformed_names() { + for name in [ + format!("{HASH}.webp"), + format!("{HASH}.jpg"), + format!("ext-{UUID}.webp"), + format!("ext-{UUID}"), + "ext-not-a-uuid.jpg".to_string(), + format!("{UUID}.jpg"), + ] { + assert_eq!( + ThumbAttachedImport::file_id_from_sidecar_name(&name), + None, + "must not be imported as an attached preview: {name}" + ); + } + } + + /// The sentinel must be stable: rows carrying it are how an operator + /// tells an imported preview from one with real provenance. + #[test] + fn imported_uploader_is_the_nil_sentinel() { + assert_eq!( + IMPORTED_UPLOADER.to_string(), + "00000000-0000-0000-0000-000000000000" + ); + } +} diff --git a/src/infrastructure/services/thumb_derived_import_service.rs b/src/infrastructure/services/thumb_derived_import_service.rs new file mode 100644 index 00000000..37e67200 --- /dev/null +++ b/src/infrastructure/services/thumb_derived_import_service.rs @@ -0,0 +1,949 @@ +//! `thumb_derived_import` — backfill `storage.content_derived_blobs` from the +//! on-disk thumbnail sidecars that predate it. +//! +//! Step 10 of `docs/plan/derived-blobs.md`. Every server-rendered thumbnail +//! written before `content_derived_blobs` existed lives only as +//! `{thumbnails_root}/{size}/{hash}.webp`. That is local-disk state: another +//! instance cannot see it, a backend migration does not carry it, and no +//! consistency job covers it. This job moves those bytes into the blob store +//! and records the mapping, after which the derived tier can become +//! authoritative and the sidecar can be deleted. +//! +//! **Thumbnails only.** The table also holds `kind = 'transcode'`, and those +//! need their own import — `ImageTranscodeService` already exists and caches +//! to `.transcoded/{ext}/{file_id}.{ext}`, a different tree with a different +//! key. Importing them means **re-keying** file→content, which is legitimate +//! only because a transcode is derivable from the source bytes. Separate job; +//! this one will not grow a transcode arm. +//! +//! ### Idempotent by construction +//! +//! Each file is skipped when a row already exists for its +//! `(source_hash, 'thumbnail', variant)`, and `store_derived_blob` is +//! `ON CONFLICT DO NOTHING` with a release-on-conflict underneath, so a +//! re-run cannot inflate refcounts. Re-running is the expected operator +//! behaviour — Phase 3 (deleting the sidecars) is gated on a run reporting +//! zero imported. +//! +//! ### Multi-instance caveat +//! +//! Sidecars are local. Running this on one instance migrates only that +//! instance's files, so Phase 3 must be gated on *every* instance reporting +//! an empty tail. The run history does not aggregate across instances; that +//! remains an operator responsibility. + +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use tokio::fs; + +use crate::application::ports::thumbnail_ports::{ThumbnailFormat, ThumbnailSize}; +use crate::infrastructure::scheduler::{ + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, Mutates, RecoverableJobHandler, + RunOutcome, RunStatus, record_or_log, +}; +use crate::infrastructure::services::dedup_service::DedupService; + +pub const THUMB_DERIVED_IMPORT_JOB_NAME: &str = "thumb_derived_import"; + +/// Where the legacy tree is moved when it cannot be deleted. +/// +/// Deletion is always attempted first — this is the fallback for the one +/// case `remove_dir` refuses: a file that is not a sidecar sitting in the +/// directory (Finder's `.DS_Store`, most often). What matters to the read +/// path is that `.thumbnails` stops existing, so moving the tree aside +/// achieves the same thing while preserving whatever the stray file was. +pub(crate) const PARKED_DIR_NAME: &str = ".thumbnails.migrated"; + +/// Record a sidecar deletion on the audit channel. +/// +/// Both import jobs delete user-visible files during a one-way migration, so +/// the trail has to survive the run history: findings are per-run and get +/// purged, whereas `target: "audit"` is separable and retained. If a preview +/// later turns out to be missing, this is the only record that says the +/// migration removed it, when, and on whose behalf. +/// +/// `owner` is the id the file belonged to — a `source_hash` for content-keyed +/// sidecars, a `file_id` for uploaded ones. That is the field an +/// investigation starts from, and the raw `NEW BLOB` logs cannot supply it: +/// they name the hash of the stored bytes, which is a different value from +/// the sidecar's own name. +/// +/// `reason` is a stable machine-readable key, per the convention: `imported` +/// (replaced by a verified blob), `source_gone`, `orphaned`. +pub(crate) fn audit_sidecar_deleted( + job: &str, + reason: &str, + owner: &str, + blob_hash: &str, + path: &std::path::Path, +) { + tracing::info!( + target: "audit", + event = "thumbnail.sidecar_deleted", + reason = reason, + job = job, + owner = owner, + blob_hash = blob_hash, + path = %path.display(), + "👮🏻‍♂️ migration deleted a thumbnail sidecar ({reason})", + ); +} + +/// Files handled between checkpoints. Each one is a read plus (at most) a +/// blob write, so this is deliberately smaller than a pure-DB sweep's page. +const BATCH_SIZE: usize = 100; + +/// Remove `.thumbnails/` — but only once BOTH import jobs have drained it. +/// +/// The directory is shared and each job owns half of it: hash-named +/// sidecars belong to `thumb_derived_import`, `ext-{file_id}.jpg` to +/// `thumb_attached_import`. Whichever runs first therefore finds the +/// other's files still present. +/// +/// The first version let the derived job tear down unilaterally. It ran +/// first, deleted its own sidecars, found `remove_dir` refused because the +/// `ext-*` previews were still there, and fell back to renaming the tree +/// to `.thumbnails.migrated`. The attached job then looked in +/// `.thumbnails/`, found nothing, and reported zeros — stranding the +/// user-uploaded previews, which are the one class of file here that +/// cannot be regenerated. The rename fired for exactly the wrong reason: +/// it exists for files NEITHER job claims, and it fired for the sibling's +/// work-in-progress. +/// +/// So the rule is: if anything remains that either job would claim, do +/// nothing at all and let the sibling finish. Whichever job runs last then +/// finds a genuinely empty tree and removes it, in the same boot. +/// +/// The rename survives for its original purpose only — a file no job +/// claims (Finder's `.DS_Store`) blocking `remove_dir` forever, which +/// would keep the read fallback alive on every developer machine. +pub(crate) async fn teardown_if_drained(root: &std::path::Path, job: &str, run_id: &str) { + // Already gone — an earlier run drained it. This is the END STATE, not a + // failure, and it is what every boot after the migration looks like. + // Falling through would `remove_dir` a missing directory and report + // ENOENT as "could not be removed", warning about success forever. + if fs::metadata(root).await.is_err() { + tracing::debug!( + target: "oxicloud::dedup", + event = "thumbnail.teardown_noop", + job = job, + run_id = run_id, + "no legacy sidecar directory — nothing to tear down" + ); + return; + } + + let mut claimed_remaining = 0usize; + let mut foreign_remaining = 0usize; + + for size in ThumbnailSize::all() { + let dir = root.join(size.dir_name()); + let Ok(mut entries) = fs::read_dir(&dir).await else { + continue; // already gone + }; + while let Ok(Some(entry)) = entries.next_entry().await { + match entry.file_name().to_str() { + // Either job's file. `hash_from_sidecar_name` covers the + // content-keyed sidecars, the `ext-` prefix the file-keyed + // previews; between them that is everything a migration + // still has to move. + Some(name) + if ThumbDerivedImport::hash_from_sidecar_name(name).is_some() + || name.starts_with("ext-") => + { + claimed_remaining += 1; + } + _ => foreign_remaining += 1, + } + } + } + + if claimed_remaining > 0 { + tracing::info!( + target: "oxicloud::dedup", + event = "thumbnail.teardown_deferred", + job = job, + run_id = run_id, + remaining = claimed_remaining, + "legacy sidecar directory left in place — {claimed_remaining} file(s) still \ + belong to the sibling import job, which has not finished draining them" + ); + return; + } + + for size in ThumbnailSize::all() { + let _ = fs::remove_dir(root.join(size.dir_name())).await; + } + + match fs::remove_dir(root).await { + Ok(()) => tracing::info!( + target: "oxicloud::dedup", + event = "thumbnail.root_removed", + job = job, + run_id = run_id, + path = %root.display(), + "🧹 legacy sidecar directory removed — the fallback read path is inert \ + from the next restart" + ), + Err(e) if foreign_remaining > 0 => { + // `with_file_name`, NOT `with_extension`: `.thumbnails` is all + // stem to `Path`, so `with_extension` would have produced + // `.thumbnails.thumbnails.migrated`. + let parked = root.with_file_name(PARKED_DIR_NAME); + match fs::rename(root, &parked).await { + Ok(()) => tracing::info!( + target: "oxicloud::dedup", + event = "thumbnail.root_parked", + job = job, + run_id = run_id, + to = %parked.display(), + foreign = foreign_remaining, + "🧹 legacy sidecar directory holds {foreign_remaining} file(s) no import \ + job claims — moved aside instead of deleted, so nothing of anyone \ + else's is destroyed. Safe to remove by hand." + ), + Err(e) => tracing::warn!( + target: "oxicloud::dedup", + event = "thumbnail.root_kept", + job = job, + run_id = run_id, + reason = %e, + "legacy sidecar directory neither removed nor moved aside — the \ + fallback read path stays live" + ), + } + let _ = e; + } + Err(e) => tracing::warn!( + target: "oxicloud::dedup", + event = "thumbnail.root_kept", + job = job, + run_id = run_id, + reason = %e, + "legacy sidecar directory could not be removed" + ), + } +} + +pub struct ThumbDerivedImport { + thumbnails_root: PathBuf, + dedup: Arc, +} + +impl ThumbDerivedImport { + pub fn new(thumbnails_root: PathBuf, dedup: Arc) -> Self { + Self { + thumbnails_root, + dedup, + } + } + + pub async fn register_recoverable_job( + self: Arc, + registry: &JobRegistry, + provider: &Arc, + ) -> Arc { + // Daily tick rather than manual-only. Ops cannot be relied on to + // remember a migration, and boot-time would delay readiness for a + // filesystem walk — whereas this is idempotent and resumable, so + // periodic is safe and it drains on its own. + // + // The tick does NOT delete: `repair` defaults false, so scheduled + // runs import and stop. Deletion stays a deliberate operator action, + // per no-silent-auto-repair. Once drained, a run is a `read_dir` over + // three directories that returns nothing — and after the directory is + // removed, not even that. + // On-demand, NOT periodic. + // + // `OXICLOUD_STARTUP_JOBS` runs this at boot in repair mode, and that + // is the whole migration: nothing has written a sidecar since step + // 10d2, so the tail cannot grow after startup. A daily tick could + // only ever redo work the boot run already did — and it would do it + // WITHOUT repair, so it could not even finish the job. Once drained + // it is a `read_dir` returning nothing, every day, forever. + // + // The admin trigger remains for operators who want to re-run it by + // hand, which is the case registration exists for. + registry + .register_recoverable_job(self.clone(), provider.clone(), None) + .await; + self + } + + /// The hash and format a sidecar filename names, or `None` when the file + /// is not one of ours. + /// + /// Strict, and deliberately rejects `ext-{file_id}.jpg`: those are + /// user-supplied, file-keyed bytes. Importing them here would content-key + /// them and share one user's uploaded preview onto every file with + /// identical content — the poisoning `file_attached_blobs` exists to + /// prevent. They belong to `thumb_attached_import`. That rejection + /// carries the weight now that `.jpg` is otherwise claimed, since the two + /// jobs would otherwise both want it. + /// + /// Returns the format too, because the row + /// key needs both since migration `20261022000000`. + /// + /// Both codecs are claimed. `persist_rendered` writes + /// `{hash}.{format.ext()}`, so any client that does not advertise WebP + /// leaves `{hash}.jpg` on disk. While the derived tier was WebP-only + /// those were unmigratable by design; now that `variant` carries the + /// format they are ordinary content, and skipping them would leave + /// `.thumbnails/` permanently non-empty — which is the signal step 10e + /// gates the fallback removal on. + fn hash_from_sidecar_name(name: &str) -> Option<(&str, ThumbnailFormat)> { + let (stem, format) = ThumbnailFormat::ALL + .iter() + .find_map(|f| name.strip_suffix(&format!(".{}", f.ext())).map(|s| (s, *f)))?; + if stem.len() != 64 || !stem.chars().all(|c| c.is_ascii_hexdigit()) { + return None; + } + Some((stem, format)) + } + + /// Delete a sidecar, but only after proving the blob that replaced it can + /// actually be read back. + /// + /// The verification is the whole point. `store_derived_blob` reporting + /// success is not proof the bytes are retrievable — a backend that + /// accepted a write it cannot serve would otherwise have the last copy + /// deleted on top of it. This is a migration, and the difference between + /// a migration and a data-loss bug is exactly this read. + /// + /// Length is compared rather than full bytes: it catches the realistic + /// failures (absent, empty, truncated) without a second full read of the + /// sidecar, which the already-imported path would otherwise need. + /// + /// Returns whether the file was removed. A failed verification leaves the + /// sidecar in place — the run reports it and the next one retries, which + /// is the safe direction. + /// Shared with `thumb_attached_import` rather than copied into it: both + /// jobs delete a sidecar only after proving its replacement is readable, + /// and two copies of that rule would be two chances to weaken one. + pub(crate) async fn verify_and_unlink( + dedup: &DedupService, + job: &str, + owner: &str, + stored_hash: &str, + path: &std::path::Path, + ) -> bool { + // Compare CONTENT, not length. + // + // This is the only thing standing between a storage bug and + // permanent loss — `thumb_attached_import` deletes user-uploaded + // previews that have no render path to rebuild them, and with the + // startup-job default it does so on first boot after an upgrade, + // in every deployment at once. A guard that load-bearing should + // prove the bytes are the bytes. + // + // Length alone did not. A blob of the right size and the wrong + // content passed: a key-mapping bug handing back another file's + // preview at the same length would have deleted the original and + // kept the impostor, and thumbnails cluster tightly enough in size + // for that to be a real coincidence rather than a theoretical one. + // + // Re-reading the sidecar costs a few KB of I/O, once per file ever + // migrated. The import path already has these bytes in hand, but + // taking them as an argument would leave the already-imported path + // (which has no bytes, only a file) on a weaker check — one code + // path, one guarantee. + let Ok(sidecar) = fs::read(path).await else { + return false; + }; + // `read_blob_bytes` streams from the backend, reassembling chunks + // if the blob is chunked — no cache sits in front of it, so this + // proves durability and not merely that a write was acknowledged. + let Ok(stored) = dedup.read_blob_bytes(stored_hash).await else { + return false; + }; + if stored.is_empty() || stored.as_ref() != sidecar.as_slice() { + return false; + } + if fs::remove_file(path).await.is_err() { + return false; + } + audit_sidecar_deleted(job, "imported", owner, stored_hash, path); + true + } + + /// Sorted sidecar filenames for one size directory. + /// + /// Sorted so the cursor is meaningful: resume skips everything at or + /// before it, which only works over a stable order. + /// + /// Takes the root rather than reading `self`, so the walk — the half that + /// decides which files this job claims, and therefore which keying they + /// get — is testable against a temp directory with no database in sight. + pub(crate) async fn sidecar_names(root: &std::path::Path, size: ThumbnailSize) -> Vec { + let dir = root.join(size.dir_name()); + let Ok(mut entries) = fs::read_dir(&dir).await else { + return Vec::new(); + }; + let mut names = Vec::new(); + while let Ok(Some(entry)) = entries.next_entry().await { + if let Some(name) = entry.file_name().to_str() + && Self::hash_from_sidecar_name(name).is_some() + { + names.push(name.to_string()); + } + } + names.sort(); + names + } +} + +#[async_trait] +impl RecoverableJobHandler for ThumbDerivedImport { + fn name(&self) -> &str { + THUMB_DERIVED_IMPORT_JOB_NAME + } + + fn description(&self) -> &'static str { + "Migrates server-rendered thumbnails from the legacy .thumbnails/ \ + directory into content-addressed blob storage. Local-disk sidecars \ + are invisible to other instances and are not carried by a backend \ + migration; importing them is what lets that directory be deleted." + } + + /// `Always`: a plain run inserts rows and writes blobs. Repair-capable on + /// top of that, which is why the two are independent. + fn mutates(&self) -> Mutates { + Mutates::Always + } + + fn repair_description(&self) -> Option<&'static str> { + Some( + "Also DELETES each sidecar once its replacement has been read \ + back from blob storage, and removes the directory when empty. \ + Files whose source no longer exists are deleted without a \ + readback — they cannot be imported and nothing can reference \ + them. Irreversible.", + ) + } + + async fn count_total(&self) -> Option { + let mut total = 0u64; + for size in ThumbnailSize::all() { + total += Self::sidecar_names(&self.thumbnails_root, *size) + .await + .len() as u64; + } + Some(total) + } + + async fn run_resumable( + &self, + store: &dyn JobStore, + args: &JobRunArgs, + resume_cursor: Option>, + ) -> RunOutcome { + // `?repair=true` opts into deleting each sidecar once it has been + // imported AND read back. Off by default, matching the house rule + // that a job does not mutate on its default setting — early runs + // import only, so an operator can inspect before committing. + // + // Deleting from the job rather than from a later release is what + // makes the migration self-draining: sidecars are LOCAL disk, so no + // release can know whether every instance has finished, whereas each + // instance draining itself needs no coordination at all. + let delete_imported = args.repair; + // Cursor is `{size_dir}/{filename}` — the last file completed. Sizes + // are walked in `ThumbnailSize::all()` order, and names are sorted + // within each, so the pair totally orders the walk. + let cursor: Option = match resume_cursor { + None => None, + Some(b) if b.is_empty() => None, + Some(b) => match String::from_utf8(b) { + Ok(s) => Some(s), + Err(e) => { + return RunOutcome::Failed { + message: format!("invalid cursor: not valid UTF-8: {e}"), + }; + } + }, + }; + + let mut imported = 0u64; + let mut already = 0u64; + let mut failed = 0u64; + let mut deleted = 0u64; + let mut unverified = 0u64; + let mut dead_source = 0u64; + let mut since_checkpoint = 0usize; + // The DIRECTORY is `{size}` on disk; the VARIANT is `{size}.{ext}` + // since migration `20261022000000`. Conflating them is a real trap: + // using the variant as a path yields `.thumbnails/preview.webp/…`, + // which does not exist, so every file reads as unreadable and nothing + // imports. The variant is therefore built per FILE, from the format + // its extension names, not once per size. + for size in ThumbnailSize::all() { + let dir_name = size.dir_name(); // on-disk directory + for name in Self::sidecar_names(&self.thumbnails_root, *size).await { + // Cursor position uses the DIRECTORY, so a run paused before + // this change resumes at the same place. + let position = format!("{dir_name}/{name}"); + + // Resume: everything at or before the cursor is done. + if let Some(c) = &cursor + && position.as_str() <= c.as_str() + { + continue; + } + + match store.status().await { + Ok(RunStatus::CancelRequested) => { + return RunOutcome::Paused { + cursor: position.into_bytes(), + }; + } + Ok(_) => {} + Err(e) => { + return RunOutcome::Failed { + message: format!("status poll: {e}"), + }; + } + } + + let Some((hash, format)) = Self::hash_from_sidecar_name(&name) else { + continue; + }; + // Both derived from the file's OWN extension, so a `.jpg` + // sidecar becomes a JPEG row rather than being mislabelled + // WebP — which would serve the wrong codec to anyone the read + // path then matched it for. + let variant = format!("{dir_name}.{}", format.ext()); + let content_type = format.mime(); + + // Already mapped — the common case on a re-run, and the + // reason this job is safe to trigger repeatedly. + // + // Deletion applies here too, not just to fresh imports: a run + // without `repair` leaves the sidecar behind, and a later run + // with it would otherwise classify the file as "already + // imported" and never drain it. Import-then-enable-deletion + // is the expected operator sequence, so this is the common + // path, not an edge case. + if let Some(existing) = self + .dedup + .find_derived_blob(hash, "thumbnail", &variant) + .await + { + already += 1; + if delete_imported { + let path = self.thumbnails_root.join(dir_name).join(&name); + if Self::verify_and_unlink( + &self.dedup, + THUMB_DERIVED_IMPORT_JOB_NAME, + hash, + &existing.blob_hash, + &path, + ) + .await + { + deleted += 1; + } else { + unverified += 1; + record_or_log( + store, + THUMB_DERIVED_IMPORT_JOB_NAME, + "sidecar_delete_unverified", + "anomaly", + None, + serde_json::json!({ + "path": position, + "hash": hash, + "note": "derived blob did not read back; sidecar kept", + }), + ) + .await; + } + } + } else if !self.dedup.blob_exists(hash).await { + // The source is gone, so this sidecar cannot be imported: + // a mapping to a dead source is precisely the orphan row + // `store_derived_blob` now refuses, because nothing would + // ever reap that hash again and the row would pin its + // artifact forever. + // + // Checked BEFORE the read and the blob write, not after. + // Without this the refusal still happens, but only once + // the bytes have been stored — so every run writes a blob + // and immediately deletes its manifest again, per dead + // sidecar, forever. On a real install where `.thumbnails/` + // has outlived years of deleted files, that is most of + // them. + // + // It also matters for the tail: these files are + // unimportable by definition, so a run that keeps + // rediscovering them never reports zero and step 10e's + // gate never opens. Under `repair` they are deleted — + // safe, and the only unlink here that needs no readback, + // since there is nothing to read back and nothing to + // regenerate from. + dead_source += 1; + let mut removed = false; + if delete_imported { + let path = self.thumbnails_root.join(dir_name).join(&name); + if fs::remove_file(&path).await.is_ok() { + deleted += 1; + removed = true; + // Audited explicitly: this unlink bypasses + // verify_and_unlink, which has nothing to verify + // against here. + audit_sidecar_deleted( + THUMB_DERIVED_IMPORT_JOB_NAME, + "source_gone", + hash, + "-", + &path, + ); + } + } + // Recorded in BOTH modes. The finding used to be the + // `else` of the deletion, so a repair run unlinked files + // and reported a clean sweep — the audit stream held the + // only trace, and the run drawer an operator actually + // looks at said zero. A deletion is the outcome most + // worth a finding, not least. + record_or_log( + store, + THUMB_DERIVED_IMPORT_JOB_NAME, + "sidecar_source_gone", + // `anomaly` in both modes — it is what the panel + // renders as "notices", and `detail.deleted` carries + // whether the run left the sidecar alone or removed + // it. A separate severity for the deleted case would + // render identically and split one badge across two + // values. + "anomaly", + None, + serde_json::json!({ + "path": position, + "source_hash": hash, + "deleted": removed, + "note": if removed { + "source Blob no longer exists; sidecar was unimportable and \ + has been deleted" + } else { + "source Blob no longer exists; the thumbnail is unimportable \ + and is deleted on a repair run" + }, + }), + ) + .await; + } else { + let path = self.thumbnails_root.join(dir_name).join(&name); + match fs::read(&path).await { + Ok(data) => { + match self + .dedup + .store_derived_blob( + hash, + "thumbnail", + &variant, + content_type, + Bytes::from(data), + ) + .await + { + Ok(derived_hash) => { + imported += 1; + if delete_imported { + if Self::verify_and_unlink( + &self.dedup, + THUMB_DERIVED_IMPORT_JOB_NAME, + hash, + &derived_hash, + &path, + ) + .await + { + deleted += 1; + } else { + unverified += 1; + record_or_log( + store, + THUMB_DERIVED_IMPORT_JOB_NAME, + "sidecar_delete_unverified", + "anomaly", + None, + serde_json::json!({ + "path": position, + "hash": hash, + "note": "derived blob did not read back; sidecar kept", + }), + ) + .await; + } + } + } + Err(e) => { + failed += 1; + record_or_log( + store, + THUMB_DERIVED_IMPORT_JOB_NAME, + "thumbnail_import_failed", + "anomaly", + None, + serde_json::json!({ + "path": position, + "hash": hash, + "error": format!("{e}"), + "note": "sidecar left in place; safe to re-run", + }), + ) + .await; + } + } + } + Err(e) => { + // Unreadable, or removed between listing and read + // (a concurrent GC unlink). Neither is fatal. + failed += 1; + record_or_log( + store, + THUMB_DERIVED_IMPORT_JOB_NAME, + "thumbnail_unreadable", + "anomaly", + None, + serde_json::json!({ + "path": position, + "error": format!("{e}"), + }), + ) + .await; + } + } + } + + since_checkpoint += 1; + if since_checkpoint >= BATCH_SIZE { + if let Err(e) = store + .checkpoint(position.clone().into_bytes(), since_checkpoint as u64) + .await + { + return RunOutcome::Failed { + message: format!("checkpoint: {e}"), + }; + } + since_checkpoint = 0; + } + } + } + + // Flush the tail. The loop only checkpoints on a full batch, so the + // remainder after the last one was never counted — a run of fewer + // than BATCH_SIZE files reported `scanned_count: 0` against a known + // total and left the admin progress bar at zero for its whole life. + // Same fix in both imports and in transcode_import. + if since_checkpoint > 0 + && let Err(e) = store.checkpoint(Vec::new(), since_checkpoint as u64).await + { + return RunOutcome::Failed { + message: format!("final checkpoint: {e}"), + }; + } + + // Remove the size directories once genuinely empty, because ABSENCE + // is what step 10e gates the fallback removal on — not emptiness. + // Empty is momentary: an on-demand render can repopulate it the next + // second. Absence is one-way, and far cheaper to test besides — one + // `stat` versus an opendir/readdir/closedir. + // + // `remove_dir` refuses a non-empty directory, so this needs no + // emptiness check of its own and cannot race a concurrent write into + // deleting live files. + if delete_imported { + teardown_if_drained( + &self.thumbnails_root, + THUMB_DERIVED_IMPORT_JOB_NAME, + &store.run_id().to_string(), + ) + .await; + } + + tracing::info!( + target: "oxicloud::dedup", + event = "thumb_derived_import.completed", + run_id = %store.run_id(), + imported = imported, + already_present = already, + failed = failed, + deleted = deleted, + unverified = unverified, + dead_source = dead_source, + "thumb_derived_import: {imported} imported, {already} already present, \ + {failed} failed, {deleted} sidecar(s) deleted, {unverified} kept unverified, \ + {dead_source} skipped (source gone)" + ); + + // Surfaced on the run row, not just in the process log. A repair run + // that unlinks hundreds of files while reporting only a finding + // total tells an operator nothing about what it did with them. + RunOutcome::completed_with(serde_json::json!({ + "imported": imported, + "already_present": already, + "deleted": deleted, + "unverified": unverified, + "dead_source": dead_source, + "failed": failed, + })) + } +} + +#[cfg(test)] +// `pub(crate)` so the attached import's test can reuse `legacy_tree`. Both +// jobs walk ONE directory, so the property worth asserting spans them — that +// together they claim every sidecar exactly once — and that needs a shared +// fixture rather than two that can drift apart. +pub(crate) mod tests { + use super::*; + + const H: &str = "0a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9"; + /// A second hash, for the JPEG sidecar in `legacy_tree`. + const H2: &str = "c222222222222222222222222222222222222222222222222222222222222222"; + + /// The park path must be a SIBLING of `.thumbnails`, not a suffixed + /// child of its name. + /// + /// `Path::with_extension` looks right and is wrong here: a leading-dot + /// name has no extension as far as `Path` is concerned — `.thumbnails` + /// is entirely stem — so `with_extension("thumbnails.migrated")` + /// yields `.thumbnails.thumbnails.migrated`. The rename would still + /// have "worked", leaving a directory nobody documented and an + /// operator hunting for the name the runbook promised. + #[test] + fn parked_directory_is_a_sibling_named_thumbnails_migrated() { + let root = std::path::Path::new("/srv/storage/.thumbnails"); + assert_eq!( + root.with_file_name(PARKED_DIR_NAME), + std::path::Path::new("/srv/storage/.thumbnails.migrated"), + ); + } + + /// BOTH codecs are claimed, and the format comes from the extension. + /// + /// `.jpg` was previously rejected here, which was correct only while the + /// derived tier was WebP-only. Once `variant` carried the format + /// (migration `20261022000000`) a JPEG sidecar became ordinary content, + /// and leaving it unclaimed would keep `.thumbnails/` permanently + /// non-empty — the very signal step 10e gates on. + #[test] + fn accepts_both_codecs_and_reports_the_format() { + assert_eq!( + ThumbDerivedImport::hash_from_sidecar_name(&format!("{H}.webp")), + Some((H, ThumbnailFormat::Webp)) + ); + assert_eq!( + ThumbDerivedImport::hash_from_sidecar_name(&format!("{H}.jpg")), + Some((H, ThumbnailFormat::Jpeg)), + "a JPEG sidecar must import, and as JPEG — labelling it WebP \ + would serve the wrong codec" + ); + } + + /// A legacy `.thumbnails` tree as it exists before the migration: both + /// sidecar shapes side by side in the same size directory, which is + /// exactly how they are written today. + /// + /// Returns the temp dir — the caller must hold it, or the directory is + /// removed while the test is still reading it. + pub(crate) async fn legacy_tree() -> tempfile::TempDir { + let tmp = tempfile::tempdir().expect("create temp dir"); + for size in ThumbnailSize::all() { + let dir = tmp.path().join(size.dir_name()); + tokio::fs::create_dir_all(&dir).await.unwrap(); + // Server-rendered, content-keyed. `b` sorts after `0a…`, so the + // pair also proves the listing is ordered rather than incidental. + tokio::fs::write(dir.join(format!("{H}.webp")), b"webp") + .await + .unwrap(); + tokio::fs::write( + dir.join("b111111111111111111111111111111111111111111111111111111111111111.webp"), + b"webp2", + ) + .await + .unwrap(); + // User-uploaded, file-keyed. + tokio::fs::write( + dir.join("ext-3f2b1c00-1111-2222-3333-444455556666.jpg"), + b"jpeg", + ) + .await + .unwrap(); + // Neither: a stray file that must be claimed by no one. + // Server-rendered JPEG: what a client not advertising WebP + // leaves behind. Claimed by the derived import, and must not be + // confused with the `ext-` upload above despite sharing an + // extension. + tokio::fs::write(dir.join(format!("{H2}.jpg")), b"jpeg") + .await + .unwrap(); + tokio::fs::write(dir.join("README.txt"), b"nope") + .await + .unwrap(); + } + tmp + } + + /// The migration's core invariant: this job claims the content-keyed + /// sidecars and *only* those, leaving the uploaded previews for + /// `thumb_attached_import`. Getting this wrong content-keys user-supplied + /// bytes, which shares one user's preview onto every file with identical + /// content. + #[tokio::test] + async fn walk_claims_only_content_keyed_sidecars_in_sorted_order() { + let tmp = legacy_tree().await; + let names = ThumbDerivedImport::sidecar_names(tmp.path(), ThumbnailSize::Preview).await; + + assert_eq!( + names, + vec![ + format!("{H}.webp"), + "b111111111111111111111111111111111111111111111111111111111111111.webp".to_string(), + format!("{H2}.jpg"), + ], + "must claim every content-keyed sidecar of EITHER codec, sorted, \ + and nothing else" + ); + } + + /// A missing size directory is normal on a fresh install and must not + /// abort the walk — the job simply has nothing to import. + #[tokio::test] + async fn missing_size_directory_yields_no_work() { + let tmp = tempfile::tempdir().expect("create temp dir"); + assert!( + ThumbDerivedImport::sidecar_names(tmp.path(), ThumbnailSize::Icon) + .await + .is_empty() + ); + } + + /// `ext-` files are user-supplied and file-keyed. Importing one here + /// would content-key it and share it across every file with identical + /// content — the exact poisoning the table split prevents. + #[test] + fn rejects_external_and_malformed_names() { + for name in [ + // `ext-` prefixed: user-supplied and file-keyed, whatever the + // extension. Now that .jpg is otherwise claimed, this is the case + // that keeps the two jobs disjoint. + format!("ext-{H}.jpg"), + "ext-3f2b1c00-0000-0000-0000-000000000000.jpg".to_string(), + format!("{}.webp", &H[..63]), + H.to_string(), + "junk.webp".to_string(), + "junk.jpg".to_string(), + ] { + assert_eq!( + ThumbDerivedImport::hash_from_sidecar_name(&name), + None, + "must not be imported as a derived thumbnail: {name}" + ); + } + } +} diff --git a/src/infrastructure/services/thumbnail_service.rs b/src/infrastructure/services/thumbnail_service.rs index 09b36a32..4142d2e1 100644 --- a/src/infrastructure/services/thumbnail_service.rs +++ b/src/infrastructure/services/thumbnail_service.rs @@ -17,6 +17,7 @@ use rayon::prelude::*; */ use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::fs; use tokio::sync::Semaphore; @@ -59,6 +60,22 @@ impl ThumbnailSize { } } + /// The `content_derived_blobs.variant` value for this size and format. + /// + /// One place builds the string, because it is a primary-key component: a + /// writer and a reader that disagree do not fail loudly, they simply + /// never find each other's rows — the read falls back to the sidecar and + /// the derived tier silently looks empty. + /// + /// The format term is what lets one source hold both codecs at a size. + /// Without it a JPEG request matched the WebP row and would be served the + /// wrong codec, which is why the step-10c read flip had to be gated to + /// WebP and why JPEG clients could never leave the sidecar. See migration + /// `20261022000000`. + pub fn derived_variant(&self, format: ThumbnailFormat) -> String { + format!("{}.{}", self.dir_name(), format.ext()) + } + /// Get all thumbnail sizes pub fn all() -> &'static [ThumbnailSize] { &[ @@ -69,15 +86,60 @@ impl ThumbnailSize { } } -/// Cache key for thumbnails. Includes `format` so WebP and the JPEG fallback for -/// the same (file_id, size) are distinct entries (no cross-format collision). +/// Cache key for the in-RAM thumbnail tier (moka). Includes `format` so WebP +/// and the JPEG fallback for the same (content, size) are distinct entries +/// (no cross-format collision). +/// +/// Not to be confused with the two other caches on this path: the sidecar +/// files under `thumbnails_root` (already blob-hash keyed), and +/// `CachedBlobBackend`, the on-disk LRU in front of a remote blob backend +/// that only comes into play when a derived blob is read through the dedup +/// stack. #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct ThumbnailCacheKey { - file_id: String, + /// Content hash for rendered thumbnails; `ext-{file_id}` for + /// client-uploaded video frames, which really are per-file. + /// + /// Keying on the hash rather than the file id is what makes the tier + /// coherent with the HTTP ETag. When a file's content is replaced its + /// id survives, so a file-keyed entry stayed valid-looking and had to + /// be invalidated explicitly — which `on_file_updated` does from a + /// spawned task, leaving a window where the response carried the NEW + /// ETag over the OLD bytes. Since the response is `immutable` with a + /// one-year max-age, a client landing in that window cached stale + /// bytes permanently. Content keying removes the window rather than + /// narrowing it: new content is a different key, so it cannot hit. + /// + /// It also stops N copies of one photo occupying N entries for + /// identical bytes. + /// + /// The two namespaces cannot collide: hashes are 64 hex characters, + /// and the external form carries an `ext-` prefix and a UUID. + id: String, size: ThumbnailSize, format: ThumbnailFormat, } +impl ThumbnailCacheKey { + /// Rendered thumbnail — keyed by the source content hash. + fn content(hash: &str, size: ThumbnailSize, format: ThumbnailFormat) -> Self { + Self { + id: hash.to_string(), + size, + format, + } + } + + /// Client-uploaded video frame — genuinely per-file, always JPEG. + fn external(file_id: &str, size: ThumbnailSize) -> Self { + Self { + id: format!("ext-{file_id}"), + size, + format: ThumbnailFormat::Jpeg, + } + } +} + /// Maximum pixel count before rejecting decode (50 megapixels → ~200 MB RGBA). /// Images above this are silently skipped — protects against single-image OOM. const MAX_DECODE_PIXELS: u64 = 50_000_000; @@ -138,6 +200,15 @@ pub struct ThumbnailService { /// Timeout for thumbnail generation operations to prevent hanging on large images. /// Defaults to 30 seconds. generation_timeout: Duration, + /// Whether the legacy sidecar tier still exists on disk, probed once by + /// [`Self::initialize`]. `false` short-circuits every fallback read + /// without a syscall. + /// + /// Starts `true` so a service used without `initialize()` (tests, and any + /// future construction path) keeps the old behaviour: fall back and let + /// the open fail. Failing open is the safe direction — the wrong value + /// costs syscalls, the opposite would hide sidecars that are still there. + legacy_sidecars: AtomicBool, } impl ThumbnailService { @@ -178,22 +249,87 @@ impl ThumbnailService { max_cache_bytes: max_cache_bytes as u64, decode_semaphore: Arc::new(Semaphore::new(max_concurrent_decodes())), generation_timeout: generation_timeout.unwrap_or(Duration::from_secs(30)), + legacy_sidecars: AtomicBool::new(true), } } - /// Initialize the thumbnail directories + /// Probe the legacy sidecar tier and record whether it still holds + /// anything. + /// + /// **This no longer creates the directories.** It used to `create_dir_all` + /// every size directory at boot, which silently undid the migration: the + /// import job removes them once drained, the next restart put them back, + /// and the absence step 10e gates on could never be reached. Nothing has + /// written a sidecar since step 10d2, so there is nothing to create them + /// for. + /// + /// One `stat` on the root. The import job guarantees that is enough: it + /// removes the directory once drained, and when something unrelated + /// keeps `remove_dir` from succeeding — Finder's `.DS_Store`, typically + /// — it renames the tree to `.thumbnails.migrated` rather than leaving + /// it in place. So `.thumbnails` existing always means "there may be + /// sidecars under here", and a stray file cannot pin the fallback open. + /// + /// Result is cached for the process lifetime. It can only be stale in the + /// harmless direction: a drain completing mid-life leaves the flag `true` + /// until restart, which costs the same failed opens as today. It never + /// goes `false` while sidecars remain. pub async fn initialize(&self) -> std::io::Result<()> { - for size in ThumbnailSize::all() { - let dir = self.thumbnails_root.join(size.dir_name()); - fs::create_dir_all(&dir).await?; + let present = fs::metadata(&self.thumbnails_root).await.is_ok(); + self.legacy_sidecars.store(present, Ordering::Relaxed); + + // Asymmetric on purpose. "Present" is actionable and temporary — it + // names the two jobs that clear it and stops appearing once they + // have. "Absent" is the steady state of every drained deployment + // forever, so at info it would be pure boot noise. + if present { + tracing::info!( + target: "oxicloud::thumbnails", + event = "thumbnail.legacy_tier_present", + root = ?self.thumbnails_root, + "🖼️ legacy sidecar tier present — reads fall back to it. Run \ + thumb_derived_import and thumb_attached_import with ?repair=true \ + to drain it." + ); + } else { + tracing::debug!( + target: "oxicloud::thumbnails", + event = "thumbnail.legacy_tier_absent", + root = ?self.thumbnails_root, + "🖼️ no legacy sidecar tier — fallback reads are skipped entirely" + ); } - tracing::info!( - "🖼️ Thumbnail service initialized at {:?}", - self.thumbnails_root - ); Ok(()) } + /// Whether the legacy sidecar tier is worth touching at all. + /// + /// This is the whole of step 10e. Removing the fallback in a release was + /// never workable: sidecars are local disk, so no release can know that + /// every instance has drained. Making the path self-disabling costs one + /// relaxed atomic load and needs no coordination — once a deployment has + /// drained, the code is inert and can be deleted whenever, or never. + fn legacy_tier_active(&self) -> bool { + self.legacy_sidecars.load(Ordering::Relaxed) + } + + /// Read a legacy sidecar, or `None` when the tier is inert. + /// + /// Every sidecar read goes through here so the guard exists once rather + /// than at each of the dozen sites that used to build a path and read it. + async fn read_sidecar(&self, path: &Path) -> Option { + if !self.legacy_tier_active() { + return None; + } + fs::read(path).await.ok().map(Bytes::from) + } + + /// Presence test with the same guard — for the paths that only need to + /// know whether a sidecar is there. + async fn sidecar_exists(&self, path: &Path) -> bool { + self.legacy_tier_active() && fs::metadata(path).await.is_ok() + } + /// Check if a file is an image that can have thumbnails pub fn is_supported_image(mime_type: &str) -> bool { matches!( @@ -216,6 +352,71 @@ impl ThumbnailService { .join(format!("{}.{}", blob_hash, format.ext())) } + /// Persist a freshly rendered thumbnail to every durable tier. + /// + /// **One place that knows what persisting a thumbnail means.** Before + /// this, four render paths each wrote the sidecar and exactly one also + /// recorded the `content_derived_blobs` row, so an on-demand render — a + /// cache miss, a size never generated, an evicted sidecar — produced + /// state the migration could never see. That is not untidy, it breaks + /// the migration's premise: `thumb_derived_import` would never reach an + /// empty tail, and the gate for deleting the sidecar would never open. + /// + /// Scope is deliberately *durable* tiers only. The moka entry is left to + /// callers because several persist through `cache.entry().or_insert_with`, + /// which already owns the insert; doing it here too would write twice. + /// + /// Dual-write is the interim setting, not the destination. Once the + /// derived tier is authoritative and the imports have drained, dropping + /// the sidecar becomes a one-line change *here* rather than four edits + /// spread across the file — which is the point of consolidating first. + /// + /// Both writes are best-effort and logged: the bytes are already rendered + /// and about to be served, so a persistence failure must cost a + /// re-render later, never the response now. `dedup: None` means + /// sidecar-only — a caller that could not supply one, which is visible at + /// the call site rather than hidden as a missing line. + async fn persist_rendered( + &self, + blob_hash: &str, + size: ThumbnailSize, + format: ThumbnailFormat, + bytes: &Bytes, + dedup: Option<&DedupService>, + ) { + // Step 10d2: the sidecar write is GONE. The derived tier is the only + // durable home for a rendered thumbnail now. + // + // Safe because the read flip landed first: reads already prefer the + // derived tier, so nothing depended on this write to be found. And a + // failure below costs a re-render rather than data — a rendered + // thumbnail is regenerable by definition, which is exactly why this + // side could stop before the uploaded one. + // + // Existing sidecars are untouched. They stay readable through the + // fallback tier until the import drains them, so a box that has not + // run the job yet loses nothing. + if let Some(dedup) = dedup + && let Err(e) = dedup + .store_derived_blob( + blob_hash, + "thumbnail", + &size.derived_variant(format), + format.mime(), + bytes.clone(), + ) + .await + { + tracing::warn!( + target: "oxicloud::dedup", + error = %e, + "failed to record derived blob for {} {:?}", + &blob_hash[..blob_hash.len().min(12)], + size, + ); + } + } + /// Get a thumbnail, generating it if needed. /// /// # Arguments @@ -234,15 +435,12 @@ impl ThumbnailService { format: ThumbnailFormat, original_path: &Path, ) -> Result { - let cache_key = ThumbnailCacheKey { - file_id: file_id.to_string(), - size, - format, - }; + let cache_key = ThumbnailCacheKey::content(blob_hash, size, format); let thumb_path = self.get_thumbnail_path(blob_hash, size, format); let original_owned = original_path.to_path_buf(); let file_id_owned = file_id.to_string(); + let blob_hash_owned = blob_hash.to_string(); // Moka's entry().or_insert_with() guarantees that for the same key // only ONE init closure runs; concurrent callers await the same @@ -252,24 +450,33 @@ impl ThumbnailService { .entry(cache_key) .or_insert_with(async { // 1. Try loading from disk - if let Ok(data) = fs::read(&thumb_path).await { + if let Some(bytes) = self.read_sidecar(&thumb_path).await { tracing::debug!( "💾 Thumbnail loaded from disk: {} {:?}", file_id_owned, size ); - return Bytes::from(data); + return bytes; } // 2. Generate thumbnail (CPU-bound, runs in spawn_blocking) tracing::info!("🎨 Generating thumbnail: {} {:?}", file_id_owned, size); match self.generate_thumbnail(&original_owned, size, format).await { Ok(bytes) => { - // Save to disk (best-effort — don't fail the request) - if let Some(parent) = thumb_path.parent() { - let _ = fs::create_dir_all(parent).await; - } - let _ = fs::write(&thumb_path, &bytes).await; + // `None` — sidecar-only, and that is acceptable here + // ONLY because this path is production-unreachable: + // its sole caller is the `ThumbnailPort` impl, and + // nothing holds a `dyn ThumbnailPort` (checked). Live + // renders go through `get_thumbnail_from_blob`, which + // dual-writes. + // + // If this ever gains a real caller it must take a + // `DedupService` first, or it reopens the gap + // `persist_rendered` exists to close: sidecar-only + // output the import can never see, so the tail never + // empties. + self.persist_rendered(&blob_hash_owned, size, format, &bytes, None) + .await; bytes } Err(e) => { @@ -312,34 +519,42 @@ impl ThumbnailService { format: ThumbnailFormat, original_data: Bytes, ) -> Result { - let cache_key = ThumbnailCacheKey { - file_id: file_id.to_string(), - size, - format, - }; + let cache_key = ThumbnailCacheKey::content(blob_hash, size, format); let thumb_path = self.get_thumbnail_path(blob_hash, size, format); let file_id_owned = file_id.to_string(); + let blob_hash_owned = blob_hash.to_string(); let entry = self .cache .entry(cache_key) .or_insert_with(async move { - if let Ok(data) = fs::read(&thumb_path).await { + if let Some(bytes) = self.read_sidecar(&thumb_path).await { tracing::debug!( "💾 Thumbnail loaded from disk: {} {:?}", file_id_owned, size ); - return Bytes::from(data); + return bytes; } let Ok(_permit) = self.decode_semaphore.acquire().await else { tracing::warn!("Decode semaphore closed, skipping {}", file_id_owned); return Bytes::new(); }; - self.generate_and_persist(&file_id_owned, &thumb_path, size, format, original_data) - .await + // `None`: this entry point takes the original bytes directly + // and has no DedupService, so it persists sidecar-only. The + // gap is visible here rather than hidden as a missing write, + // and closing it means threading dedup in from its callers. + self.generate_and_persist( + &file_id_owned, + &blob_hash_owned, + size, + format, + original_data, + None, + ) + .await }) .await; @@ -369,11 +584,7 @@ impl ThumbnailService { format: ThumbnailFormat, dedup: Arc, ) -> Result { - let cache_key = ThumbnailCacheKey { - file_id: file_id.to_string(), - size, - format, - }; + let cache_key = ThumbnailCacheKey::content(blob_hash, size, format); let thumb_path = self.get_thumbnail_path(blob_hash, size, format); let file_id_owned = file_id.to_string(); @@ -383,13 +594,13 @@ impl ThumbnailService { .cache .entry(cache_key) .or_insert_with(async move { - if let Ok(data) = fs::read(&thumb_path).await { + if let Some(bytes) = self.read_sidecar(&thumb_path).await { tracing::debug!( "💾 Thumbnail loaded from disk: {} {:?}", file_id_owned, size ); - return Bytes::from(data); + return bytes; } let Ok(_permit) = self.decode_semaphore.acquire().await else { @@ -407,8 +618,19 @@ impl ThumbnailService { return Bytes::new(); } }; - self.generate_and_persist(&file_id_owned, &thumb_path, size, format, original_data) - .await + // The on-demand render the REST handler falls through to on a + // cache miss — the busiest path that previously wrote a + // sidecar and no row. `dedup` is already in scope here, so + // dual-writing costs nothing. + self.generate_and_persist( + &file_id_owned, + &blob_hash_owned, + size, + format, + original_data, + Some(dedup.as_ref()), + ) + .await }) .await; @@ -431,10 +653,11 @@ impl ThumbnailService { async fn generate_and_persist( &self, file_id: &str, - thumb_path: &Path, + blob_hash: &str, size: ThumbnailSize, format: ThumbnailFormat, original_data: Bytes, + dedup: Option<&DedupService>, ) -> Bytes { tracing::info!("🎨 Generating thumbnail: {} {:?}", file_id, size); match Self::generate_thumbnail_from_data( @@ -446,10 +669,8 @@ impl ThumbnailService { .await { Ok(bytes) => { - if let Some(parent) = thumb_path.parent() { - let _ = fs::create_dir_all(parent).await; - } - let _ = fs::write(&thumb_path, &bytes).await; + self.persist_rendered(blob_hash, size, format, &bytes, dedup) + .await; bytes } Err(e) => { @@ -472,20 +693,128 @@ impl ThumbnailService { /// `blob_hash` is used to locate the file on disk (dedup-aware). /// If `None`, only the in-memory cache is checked (used for video /// thumbnails where blob_hash is not yet resolved). + /// Identity of the bytes a thumbnail request will serve — the body of its + /// HTTP ETag. + /// + /// Mirrors the tier precedence in [`Self::get_cached_thumbnail`], because + /// an ETag that names a different tier than the one answering is worse + /// than a coarse one: it lets two resources serving different bytes share + /// a validator, and a shared cache may then hand either to either. + /// + /// * An **attached** blob wins, and its own hash is the identity. Nothing + /// else works: uploading a preview does not change the file's content, + /// so a source-keyed ETag would not change either — and with + /// `immutable` set, clients would never revalidate. Worse, a copy + /// inherits the source hash, so an original and a copy carrying + /// *different* uploaded previews would collide on one ETag. + /// * Otherwise the **source-keyed** form, which identifies a render of + /// known content at a known size and format. + /// + /// # Why the derived blob's own hash is NOT used yet + /// + /// It would be a better key — the hash *is* the bytes, so any change in + /// output invalidates by construction. But it cannot be resolved here + /// without flipping on the first render: the ETag is computed *before* + /// the body, so on a cache miss no `content_derived_blobs` row exists yet + /// and this returns the source-keyed form — then rendering *creates* that + /// row, and the next request resolves to the derived hash instead. The + /// validator would change as a side effect of producing the body, making + /// every first render immediately stale. + /// + /// It belongs with the read-order flip, when the derived tier becomes + /// authoritative and is populated before it is consulted. See + /// `docs/plan/derived-blobs.md`. The attached lookup above has no such + /// problem: an upload writes its row synchronously, before any read that + /// could observe it. + /// + /// Known gap: a legacy `ext-{file_id}.jpg` with no `file_attached_blobs` + /// row yet falls through to the source-keyed form, so those bytes keep + /// today's coarse validator until the import backfills the row. No worse + /// than current behaviour, and it disappears with the migration. + pub async fn thumbnail_content_id( + &self, + file_id: &str, + blob_hash: &str, + size: ThumbnailSize, + format: ThumbnailFormat, + dedup: Option<&DedupService>, + ) -> String { + if let Some(dedup) = dedup + && let Some(attached) = dedup + .find_attached_blob(file_id, "preview", size.dir_name()) + .await + { + return attached.blob_hash; + } + format!( + "thumb-{}-{}-{}", + blob_hash, + size.dir_name(), + format.as_str() + ) + } + + /// Drain a blob through the dedup stack into memory. + /// + /// Shared by the attached and derived tiers — the only difference between + /// them is which table produced the hash, so the read itself belongs in + /// one place. Returns `None` on a read fault rather than propagating: a + /// missing satellite must degrade to the next tier, never break a gallery. + async fn read_blob_to_bytes( + dedup: &DedupService, + blob_hash: &str, + file_id: &str, + size: ThumbnailSize, + ) -> Option { + use futures::StreamExt; + let mut stream = dedup.read_blob_stream(blob_hash).await.ok()?; + let mut buf = Vec::new(); + while let Some(chunk) = stream.next().await { + match chunk { + Ok(part) => buf.extend_from_slice(&part), + Err(e) => { + tracing::warn!( + target: "oxicloud::dedup", + error = %e, + "thumbnail blob read failed for {} {:?}", + file_id, + size, + ); + return None; + } + } + } + Some(Bytes::from(buf)) + } + pub async fn get_cached_thumbnail( &self, file_id: &str, blob_hash: Option<&str>, size: ThumbnailSize, format: ThumbnailFormat, + // Concrete, and optional: `ThumbnailPort` is never used as a trait + // object (checked), and `DedupPort` uses native `async fn` so it is + // not dyn-compatible anyway. `None` means sidecar-only — exactly + // today's behaviour, which is what the port impl wants. + dedup: Option<&DedupService>, ) -> Option { - // 1. Check in-memory cache - let cache_key = ThumbnailCacheKey { - file_id: file_id.to_string(), - size, - format, - }; - if let Some(bytes) = self.cache.get(&cache_key).await + // A file-specific override beats anything derived from the content, + // and that has to hold at EVERY tier — including RAM. Checking the + // content-keyed entry first would let a previously-rendered + // thumbnail shadow a preview the user has since uploaded: the render + // is cached under `content(hash)`, the upload lands under + // `external(file_id)`, and the content key would win forever. + // + // So the order is: per-file RAM, per-file disk, per-file DB, then the + // content-keyed tiers. Same precedence as the disk tiers below, just + // applied one level up. + + // 1. Per-file override in RAM (uploaded preview / video frame). + if let Some(bytes) = self + .cache + .get(&ThumbnailCacheKey::external(file_id, size)) + .await && !bytes.is_empty() { return Some(bytes); @@ -498,32 +827,111 @@ impl ThumbnailService { .thumbnails_root .join(size.dir_name()) .join(format!("ext-{}.jpg", file_id)); - if let Ok(data) = fs::read(&ext_path).await { - let bytes = Bytes::from(data); + if let Some(bytes) = self.read_sidecar(&ext_path).await { // Cache under a Jpeg-pinned key: these bytes are always JPEG, so the // key's format must describe them. Inserting under `cache_key` (whose // format is the *requested* format, possibly Webp) would store JPEG // bytes behind a Webp key — a latent cross-format invariant violation. - let ext_key = ThumbnailCacheKey { - file_id: file_id.to_string(), - size, - format: ThumbnailFormat::Jpeg, - }; + let ext_key = ThumbnailCacheKey::external(file_id, size); self.cache.insert(ext_key, bytes.clone()).await; return Some(bytes); } - // 3. Check disk for blob-hash thumbnails (needs blob_hash to locate) - let hash = blob_hash?; - let thumb_path = self.get_thumbnail_path(hash, size, format); - if let Ok(data) = fs::read(&thumb_path).await { - let bytes = Bytes::from(data); - // Populate in-memory cache for next hit - self.cache.insert(cache_key, bytes.clone()).await; - Some(bytes) - } else { - None + // 2b. Bytes the USER attached to this file, if any. + // + // Ahead of every content-derived tier below on purpose: an uploaded + // preview is an explicit choice about THIS file and must beat + // anything the server would render from its content. It is also the + // only tier a copy can inherit — the `ext-` sidecar above is keyed by + // file_id and is not copied, so without this branch a copied file + // silently falls back to a rendered thumbnail, or to none at all for + // a PDF that has no server-side render path. + if let Some(dedup) = dedup + && let Some(attached) = dedup + .find_attached_blob(file_id, "preview", size.dir_name()) + .await + && let Some(bytes) = + Self::read_blob_to_bytes(dedup, &attached.blob_hash, file_id, size).await + { + // Cached under the per-file key: these bytes belong to this file, + // not to its content, so a content key would leak them to every + // other file sharing that content — the poisoning the file-keyed + // table exists to prevent. + self.cache + .insert(ThumbnailCacheKey::external(file_id, size), bytes.clone()) + .await; + return Some(bytes); } + + // 3. Content-keyed RAM tier. Below the per-file tiers by the rule + // above; still ahead of every disk read. + // + // A caller that did not resolve the hash cannot consult it and + // falls through to disk. That is correct rather than merely + // acceptable: a file-id key here would be the stale entry content + // keying exists to avoid. Both HTTP handlers resolve the hash to + // build the ETag, so the fall-through is confined to internal + // callers that never had one. + let hash = blob_hash?; + if let Some(bytes) = self + .cache + .get(&ThumbnailCacheKey::content(hash, size, format)) + .await + && !bytes.is_empty() + { + return Some(bytes); + } + + // 4. Derived blob — the authoritative content tier (step 10c). + // + // Ahead of the sidecar now, rather than last. The sidecar is local + // disk: invisible to other instances, uncarried by a backend + // migration, uncovered by any consistency job. Reading the derived + // tier first is what lets that disk state become deletable, and it is + // not the cost it looks like — `CachedBlobBackend` gives the blob read + // a local disk cache, and moka absorbs the repeats above it. + // + // A miss FALLS THROUGH rather than ending the lookup. That is the + // whole reason this is not a two-line swap: while the imports are + // draining, most content has a sidecar and no row, and terminating + // here would return "no thumbnail" for all of it. + // + // All formats, since migration `20261022000000` put the output format + // inside `variant`. Before that, `variant` was the size alone, so a + // JPEG request matched the WebP row and would have been served the + // wrong codec — the flip had to be gated to WebP, which meant JPEG + // clients could never leave the sidecar and the sidecar could never + // be deleted. Now each codec has its own row. + if let Some(dedup) = dedup + && let Some(derived) = dedup + .find_derived_blob(hash, "thumbnail", &size.derived_variant(format)) + .await + && let Some(bytes) = + Self::read_blob_to_bytes(dedup, &derived.blob_hash, file_id, size).await + { + self.cache + .insert( + ThumbnailCacheKey::content(hash, size, format), + bytes.clone(), + ) + .await; + return Some(bytes); + } + + // 5. Blob-hash sidecar — fallback for content not yet imported, and + // the only content tier a non-WebP request can reach. + let thumb_path = self.get_thumbnail_path(hash, size, format); + if let Some(bytes) = self.read_sidecar(&thumb_path).await { + self.cache + .insert( + ThumbnailCacheKey::content(hash, size, format), + bytes.clone(), + ) + .await; + return Some(bytes); + } + + None } /// Store an externally-generated thumbnail (e.g. client-side video frame). @@ -593,24 +1001,20 @@ impl ThumbnailService { let bytes = Bytes::from(jpeg_bytes); - // External thumbnails are stored by file_id (not dedup-able) - let thumb_path = self - .thumbnails_root - .join(size.dir_name()) - .join(format!("ext-{}.jpg", file_id)); - if let Some(parent) = thumb_path.parent() { - let _ = fs::create_dir_all(parent).await; - } - fs::write(&thumb_path, &bytes) - .await - .map_err(|e| ThumbnailError::IoError(e.to_string()))?; - - // Populate in-memory cache (external thumbnails are JPEG) - let cache_key = ThumbnailCacheKey { - file_id: file_id.to_string(), - size, - format: ThumbnailFormat::Jpeg, - }; + // Step 10d2: the `ext-{file_id}.jpg` sidecar write is GONE. The + // durable store is now `file_attached_blobs`, written by the caller — + // which is why that write had to become fatal first, in the same + // change. These bytes have no server-side render path, so a + // best-effort store with no sidecar behind it would lose a user's + // upload silently. + // + // This function now re-encodes and caches; it does not persist. The + // RAM entry stays because it is what serves the request that follows, + // and the caller drops it if the durable write fails. + // + // Existing `ext-` files remain readable through the fallback tier + // until `thumb_attached_import` drains them. + let cache_key = ThumbnailCacheKey::external(file_id, size); self.cache.insert(cache_key, bytes.clone()).await; tracing::info!("✅ Stored external thumbnail: {} {:?}", file_id, size); @@ -940,7 +1344,7 @@ impl ThumbnailService { for size in ThumbnailSize::all() { let thumb_path = self.get_thumbnail_path(&blob_hash, *size, ThumbnailFormat::Webp); - if fs::metadata(&thumb_path).await.is_err() { + if !self.sidecar_exists(&thumb_path).await { ok = false; break; } @@ -951,13 +1355,10 @@ impl ThumbnailService { for size in ThumbnailSize::all() { let thumb_path = self.get_thumbnail_path(&blob_hash, *size, ThumbnailFormat::Webp); - if let Ok(data) = fs::read(&thumb_path).await { - let cache_key = ThumbnailCacheKey { - file_id: file_id.clone(), - size: *size, - format: ThumbnailFormat::Webp, - }; - self.cache.insert(cache_key, Bytes::from(data)).await; + if let Some(bytes) = self.read_sidecar(&thumb_path).await { + let cache_key = + ThumbnailCacheKey::content(&blob_hash, *size, ThumbnailFormat::Webp); + self.cache.insert(cache_key, bytes).await; } } tracing::info!( @@ -1006,22 +1407,21 @@ impl ThumbnailService { } }; - // Save each size to disk (keyed by blob_hash for dedup) - // AND populate moka (keyed by file_id for fast serving). + // Save each size to disk and populate moka — both keyed by + // blob_hash, so the two tiers agree and a copy shares them. for (size, bytes) in thumbnails { - let thumb_path = self.get_thumbnail_path(&blob_hash, size, ThumbnailFormat::Webp); - if let Some(parent) = thumb_path.parent() { - let _ = fs::create_dir_all(parent).await; - } - if let Err(e) = fs::write(&thumb_path, &bytes).await { - tracing::warn!("Failed to save thumbnail {} {:?}: {}", file_id, size, e); - } else { + // `None` — sidecar-only, acceptable for the same reason as + // `get_thumbnail`: the path variant is reached only through + // the unused `ThumbnailPort` impl. The live upload path is + // `generate_all_sizes_background_from_blob`, which carries a + // `DedupService` and dual-writes. Give this one a real caller + // and it needs one too. + self.persist_rendered(&blob_hash, size, ThumbnailFormat::Webp, &bytes, None) + .await; + { // Populate in-memory cache for instant first-hit serving - let cache_key = ThumbnailCacheKey { - file_id: file_id.clone(), - size, - format: ThumbnailFormat::Webp, - }; + let cache_key = + ThumbnailCacheKey::content(&blob_hash, size, ThumbnailFormat::Webp); self.cache.insert(cache_key, bytes).await; tracing::debug!("✅ Generated thumbnail: {} {:?}", file_id, size); } @@ -1062,7 +1462,7 @@ impl ThumbnailService { for size in ThumbnailSize::all() { let thumb_path = self.get_thumbnail_path(&blob_hash, *size, ThumbnailFormat::Webp); - if fs::metadata(&thumb_path).await.is_err() { + if !self.sidecar_exists(&thumb_path).await { ok = false; break; } @@ -1073,13 +1473,10 @@ impl ThumbnailService { for size in ThumbnailSize::all() { let thumb_path = self.get_thumbnail_path(&blob_hash, *size, ThumbnailFormat::Webp); - if let Ok(data) = fs::read(&thumb_path).await { - let cache_key = ThumbnailCacheKey { - file_id: file_id.clone(), - size: *size, - format: ThumbnailFormat::Webp, - }; - self.cache.insert(cache_key, Bytes::from(data)).await; + if let Some(bytes) = self.read_sidecar(&thumb_path).await { + let cache_key = + ThumbnailCacheKey::content(&blob_hash, *size, ThumbnailFormat::Webp); + self.cache.insert(cache_key, bytes).await; } } tracing::info!( @@ -1115,7 +1512,7 @@ impl ThumbnailService { } }; - self.render_and_persist_all_webp(&file_id, &blob_hash, original_data) + self.render_and_persist_all_webp(&file_id, &blob_hash, original_data, Some(&dedup)) .await; tracing::info!("✅ Background thumbnail generation complete: {}", file_id); @@ -1126,7 +1523,21 @@ impl ThumbnailService { /// blob_hash (disk `{hash}.webp` + moka). Shared by the image upload path and /// the video path (which passes the extracted frame as the source), so both /// produce identical, dedup-able, content-negotiable thumbnails. - async fn render_and_persist_all_webp(&self, file_id: &str, blob_hash: &str, source: Bytes) { + /// `dedup` is `Some` on every path that has a handle, which is every + /// eager background path. When present each rendered size is ALSO stored + /// as a derived blob and recorded in `storage.content_derived_blobs`. + /// + /// The sidecar write is deliberately kept: this slice fills the table + /// while reads still come from disk, so a rollback at any point leaves + /// working thumbnails and the table can be inspected against real data + /// before anything depends on it. See `docs/plan/derived-blobs.md`. + async fn render_and_persist_all_webp( + &self, + file_id: &str, + blob_hash: &str, + source: Bytes, + dedup: Option<&DedupService>, + ) { let results = tokio::task::spawn_blocking(move || { Self::render_all_thumbnails_from_data(source.as_ref(), ThumbnailFormat::Webp) }) @@ -1145,21 +1556,16 @@ impl ThumbnailService { }; for (size, bytes) in thumbnails { - let thumb_path = self.get_thumbnail_path(blob_hash, size, ThumbnailFormat::Webp); - if let Some(parent) = thumb_path.parent() { - let _ = fs::create_dir_all(parent).await; - } - if let Err(e) = fs::write(&thumb_path, &bytes).await { - tracing::warn!("Failed to save thumbnail {} {:?}: {}", file_id, size, e); - } else { - let cache_key = ThumbnailCacheKey { - file_id: file_id.to_string(), - size, - format: ThumbnailFormat::Webp, - }; - self.cache.insert(cache_key, bytes).await; - tracing::debug!("✅ Generated thumbnail: {} {:?}", file_id, size); - } + // Was the only path that wrote both tiers, with its own copy of + // the logic. Now the same `persist_rendered` every other render + // path uses, so there is one definition of what persisting means + // and the interim dual-write can be retired in one place. + self.persist_rendered(blob_hash, size, ThumbnailFormat::Webp, &bytes, dedup) + .await; + + let cache_key = ThumbnailCacheKey::content(blob_hash, size, ThumbnailFormat::Webp); + self.cache.insert(cache_key, bytes).await; + tracing::debug!("✅ Generated thumbnail: {} {:?}", file_id, size); } } @@ -1190,7 +1596,7 @@ impl ThumbnailService { let mut ok = true; for size in ThumbnailSize::all() { let p = self.get_thumbnail_path(&blob_hash, *size, ThumbnailFormat::Webp); - if fs::metadata(&p).await.is_err() { + if !self.sidecar_exists(&p).await { ok = false; break; } @@ -1200,13 +1606,10 @@ impl ThumbnailService { if all_exist { for size in ThumbnailSize::all() { let p = self.get_thumbnail_path(&blob_hash, *size, ThumbnailFormat::Webp); - if let Ok(data) = fs::read(&p).await { - let key = ThumbnailCacheKey { - file_id: file_id.clone(), - size: *size, - format: ThumbnailFormat::Webp, - }; - self.cache.insert(key, Bytes::from(data)).await; + if let Some(bytes) = self.read_sidecar(&p).await { + let key = + ThumbnailCacheKey::content(&blob_hash, *size, ThumbnailFormat::Webp); + self.cache.insert(key, bytes).await; } } return; @@ -1251,7 +1654,7 @@ impl ThumbnailService { Ok(p) => p, Err(_) => return, }; - self.render_and_persist_all_webp(&file_id, &blob_hash, frame) + self.render_and_persist_all_webp(&file_id, &blob_hash, frame, Some(&dedup)) .await; tracing::info!("✅ Video thumbnail generation complete: {}", file_id); }); @@ -1304,31 +1707,37 @@ impl ThumbnailService { Ok(tmp) } - /// Delete thumbnails for a file. + /// Delete the per-file thumbnail artifacts for a file. /// - /// Only invalidates the in-memory moka cache (keyed by file_id). - /// Disk thumbnails are keyed by blob_hash and may be shared by - /// other files with the same content — they are cleaned up via - /// `delete_blob_thumbnails` when the blob is garbage-collected. - /// Also removes any external (video-frame) thumbnails stored by file_id. + /// Only the external (video-frame) entries are file-keyed, so only those + /// are removed — from moka and from disk. Rendered thumbnails, in both + /// tiers, are keyed by blob_hash and may be shared with any other file + /// holding the same content; they are reclaimed by + /// `delete_blob_thumbnails` when the blob itself is garbage-collected. + /// + /// Content keying is also why this no longer has to win a race. When a + /// file's content is replaced the rendered entries are unreachable by + /// construction — a new hash is a new key — rather than needing explicit + /// invalidation before the next request arrives. pub async fn delete_thumbnails(&self, file_id: &str) -> Result<(), ThumbnailError> { for size in ThumbnailSize::all() { - // Remove from moka cache (lock-free invalidation) — both codecs. - for format in [ThumbnailFormat::Webp, ThumbnailFormat::Jpeg] { - let cache_key = ThumbnailCacheKey { - file_id: file_id.to_string(), - size: *size, - format, - }; - self.cache.invalidate(&cache_key).await; - } + // Only the external (per-file) entry needs invalidating. Rendered + // thumbnails are keyed by content hash, so replacing a file's + // content yields a different key and the old entry simply cannot + // be hit again — which is the point: correctness no longer depends + // on this call winning a race against the next request. And on + // deletion the entry stays valid for any other file sharing that + // content, so dropping it would only cost a re-read. + self.cache + .invalidate(&ThumbnailCacheKey::external(file_id, *size)) + .await; // Remove external (video-frame) thumbnails stored by file_id (JPEG-only) let ext_path = self .thumbnails_root .join(size.dir_name()) .join(format!("ext-{}.jpg", file_id)); - if fs::metadata(&ext_path).await.is_ok() { + if self.sidecar_exists(&ext_path).await { let _ = fs::remove_file(&ext_path).await; } } @@ -1346,7 +1755,7 @@ impl ThumbnailService { // Delete both the primary WebP and any lazily-materialized JPEG. for format in [ThumbnailFormat::Webp, ThumbnailFormat::Jpeg] { let path = self.get_thumbnail_path(blob_hash, *size, format); - if fs::metadata(&path).await.is_ok() { + if self.sidecar_exists(&path).await { let _ = fs::remove_file(&path).await; } } @@ -1567,7 +1976,10 @@ impl ThumbnailPort for ThumbnailService { blob_hash: Option<&str>, size: PortThumbnailSize, ) -> Option { - self.get_cached_thumbnail(file_id, blob_hash, size.into(), ThumbnailFormat::Webp) + // `None` — the abstract port has no DedupService handle, so it stays + // sidecar-only. Callers wanting the tier-3 fallback use the concrete + // method, which both handlers already do. + self.get_cached_thumbnail(file_id, blob_hash, size.into(), ThumbnailFormat::Webp, None) .await } @@ -1684,6 +2096,37 @@ pub enum ThumbnailError { UnsupportedFormat, } +impl ThumbnailError { + /// Is this failure a property of the CONTENT, rather than of the moment? + /// + /// Prerequisite for persisting negative verdicts to + /// `content_derived_blobs` (see `docs/plan/derived-blobs.md` §Negative + /// verdicts). Only a permanent failure may be recorded: it will give the + /// same answer forever, so remembering it saves a decode. A transient one + /// must never be recorded — the next attempt may well succeed, and a row + /// saying otherwise is silent, permanent data loss for that file. + /// + /// The asymmetry is why the default is `false`. A wrongly-persisted + /// transient marks a perfectly good image unrenderable for good; a + /// wrongly-omitted permanent merely costs a repeated decode. So anything + /// not clearly a content property is treated as transient. + /// + /// * [`Self::ImageError`] — the decoder rejected these bytes, or they + /// exceed `MAX_DECODE_PIXELS`. Both are facts about the image. + /// * [`Self::UnsupportedFormat`] — likewise. + /// * [`Self::TaskError`] — timeout, closed decode semaphore, join + /// failure. All say the machine was busy, not that the image is bad. A + /// timeout under load is the exact case that must not be cached. + /// * [`Self::IoError`] — the source could not be read. Says nothing about + /// whether it is renderable. + pub fn is_permanent(&self) -> bool { + match self { + ThumbnailError::ImageError(_) | ThumbnailError::UnsupportedFormat => true, + ThumbnailError::TaskError(_) | ThumbnailError::IoError(_) => false, + } + } +} + /// Statistics about the thumbnail cache #[derive(Debug, Clone)] pub struct ThumbnailStats { @@ -1692,6 +2135,309 @@ pub struct ThumbnailStats { pub max_cache_bytes: usize, } +#[cfg(test)] +mod tier_selection_tests { + //! Precedence in [`ThumbnailService::get_cached_thumbnail`]. + //! + //! This function produced four bugs in two days, every one of them an + //! ordering mistake rather than a logic error, and every one caught only + //! by an end-to-end run comparing bytes against something independent: + //! + //! * the content-keyed RAM entry shadowing an uploaded preview, so a PUT + //! appeared to do nothing; + //! * the same precedence being right on disk but wrong in RAM; + //! * a validator flipping because a tier was populated as a side effect + //! of producing the body; + //! * a decode error reading as "absent". + //! + //! The rule they all violate is one sentence: **a file-specific override + //! beats anything derived from the content, at every tier.** These tests + //! pin it, so the read-order flip (step 10c of + //! `docs/plan/derived-blobs.md`) is a change with a safety net rather + //! than another end-to-end guess. + //! + //! `dedup: None` throughout, which skips the two DB-backed tiers and + //! needs no database. What remains — per-file RAM, `ext-` disk, content + //! RAM, blob-hash sidecar — is exactly where the bugs were. + + use super::*; + use std::time::Duration; + + const HASH: &str = "0a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9"; + const FILE_ID: &str = "3f2b1c00-1111-2222-3333-444455556666"; + const SIZE: ThumbnailSize = ThumbnailSize::Preview; + const FMT: ThumbnailFormat = ThumbnailFormat::Webp; + + fn service(root: &std::path::Path) -> ThumbnailService { + ThumbnailService::new(root, 100, 10 * 1024 * 1024, Some(Duration::from_secs(5))) + } + + /// Seed the per-file disk tier (`ext-{file_id}.jpg`). + async fn write_ext_sidecar(root: &std::path::Path, bytes: &[u8]) { + let dir = root.join(".thumbnails").join(SIZE.dir_name()); + tokio::fs::create_dir_all(&dir).await.unwrap(); + tokio::fs::write(dir.join(format!("ext-{FILE_ID}.jpg")), bytes) + .await + .unwrap(); + } + + /// Seed the content-keyed disk tier (`{hash}.webp`). + async fn write_blob_sidecar(root: &std::path::Path, bytes: &[u8]) { + let dir = root.join(".thumbnails").join(SIZE.dir_name()); + tokio::fs::create_dir_all(&dir).await.unwrap(); + tokio::fs::write(dir.join(format!("{HASH}.{}", FMT.ext())), bytes) + .await + .unwrap(); + } + + /// `initialize()` must not recreate what the import job removed. + /// + /// It used to `create_dir_all` every size directory at boot, so a drained + /// deployment grew its `.thumbnails/` tree back on the next restart and + /// the absence the fallback gates on was unreachable. Found on a sandbox + /// where the job had removed the directories and a restart put three + /// empty ones back. + #[tokio::test] + async fn initialize_does_not_recreate_a_drained_tier() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + + svc.initialize().await.unwrap(); + + assert!( + tokio::fs::metadata(tmp.path().join(".thumbnails")) + .await + .is_err(), + "boot recreated the legacy sidecar tree" + ); + assert!( + !svc.legacy_tier_active(), + "no directories on disk, so the fallback must be inert" + ); + } + + /// The other direction: a tier that still holds sidecars stays live, or + /// the migration would strand every un-imported thumbnail. + #[tokio::test] + async fn initialize_keeps_the_fallback_when_sidecars_remain() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + write_blob_sidecar(tmp.path(), b"legacy").await; + + svc.initialize().await.unwrap(); + + assert!(svc.legacy_tier_active()); + let got = svc + .get_cached_thumbnail(FILE_ID, Some(HASH), SIZE, FMT, None) + .await; + assert_eq!(got.as_deref(), Some(&b"legacy"[..])); + } + + /// With the tier inert, a sidecar on disk is deliberately NOT served — + /// the guard short-circuits before the read. This is what makes the + /// fallback free rather than merely cheap, and it is only sound because + /// nothing has written a sidecar since step 10d2. + #[tokio::test] + async fn inert_tier_skips_the_read_entirely() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + write_blob_sidecar(tmp.path(), b"legacy").await; + svc.legacy_sidecars.store(false, Ordering::Relaxed); + + let got = svc + .get_cached_thumbnail(FILE_ID, Some(HASH), SIZE, FMT, None) + .await; + assert!(got.is_none(), "inert tier must not touch the filesystem"); + } + + /// The bug from 2026-08-25: a render cached under the content key + /// shadowed a preview the user uploaded afterwards, permanently, because + /// the content tier was consulted first. The PUT looked like a no-op. + #[tokio::test] + async fn per_file_ram_beats_content_ram() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + + svc.cache + .insert( + ThumbnailCacheKey::content(HASH, SIZE, FMT), + Bytes::from_static(b"rendered-from-content"), + ) + .await; + svc.cache + .insert( + ThumbnailCacheKey::external(FILE_ID, SIZE), + Bytes::from_static(b"uploaded-by-user"), + ) + .await; + + let got = svc + .get_cached_thumbnail(FILE_ID, Some(HASH), SIZE, FMT, None) + .await; + assert_eq!(got.as_deref(), Some(&b"uploaded-by-user"[..])); + } + + /// Same rule one tier down: the per-file file on disk must win over a + /// content-keyed entry still sitting in RAM. + #[tokio::test] + async fn ext_disk_beats_content_ram() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + + svc.cache + .insert( + ThumbnailCacheKey::content(HASH, SIZE, FMT), + Bytes::from_static(b"rendered-from-content"), + ) + .await; + write_ext_sidecar(tmp.path(), b"uploaded-on-disk").await; + + let got = svc + .get_cached_thumbnail(FILE_ID, Some(HASH), SIZE, FMT, None) + .await; + assert_eq!(got.as_deref(), Some(&b"uploaded-on-disk"[..])); + } + + /// Within the content-keyed tiers, RAM still beats disk — the ordinary + /// cache property, asserted so the flip cannot invert it by accident. + #[tokio::test] + async fn content_ram_beats_blob_sidecar() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + + write_blob_sidecar(tmp.path(), b"on-disk").await; + svc.cache + .insert( + ThumbnailCacheKey::content(HASH, SIZE, FMT), + Bytes::from_static(b"in-ram"), + ) + .await; + + let got = svc + .get_cached_thumbnail(FILE_ID, Some(HASH), SIZE, FMT, None) + .await; + assert_eq!(got.as_deref(), Some(&b"in-ram"[..])); + } + + /// The sidecar answers when nothing above it does. After step 10c this + /// becomes the *fallback* rather than the primary content tier, and this + /// test is what proves it still answers at all. + #[tokio::test] + async fn blob_sidecar_answers_when_nothing_else_does() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + + write_blob_sidecar(tmp.path(), b"only-on-disk").await; + + let got = svc + .get_cached_thumbnail(FILE_ID, Some(HASH), SIZE, FMT, None) + .await; + assert_eq!(got.as_deref(), Some(&b"only-on-disk"[..])); + } + + /// Reading the `ext-` file must cache it under the PER-FILE key. + /// + /// Under a content key those bytes would be served for every other file + /// sharing the same content — one user's uploaded preview leaking across + /// files, which is the poisoning the keying split exists to prevent. + #[tokio::test] + async fn ext_disk_read_caches_under_the_per_file_key() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + write_ext_sidecar(tmp.path(), b"uploaded").await; + + svc.get_cached_thumbnail(FILE_ID, Some(HASH), SIZE, FMT, None) + .await; + + assert_eq!( + svc.cache + .get(&ThumbnailCacheKey::external(FILE_ID, SIZE)) + .await + .as_deref(), + Some(&b"uploaded"[..]), + "must populate the per-file key" + ); + assert!( + svc.cache + .get(&ThumbnailCacheKey::content(HASH, SIZE, FMT)) + .await + .is_none(), + "must NOT populate the content key — those bytes are not derived \ + from this content and would leak to every file sharing it" + ); + } + + /// A caller with no hash cannot consult the content-keyed tiers, and must + /// fall through rather than guess. Yesterday's alternative — keying RAM + /// on `file_id` — is precisely the stale entry content-keying removed. + #[tokio::test] + async fn missing_hash_skips_content_tiers_but_still_reads_ext_disk() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + + write_blob_sidecar(tmp.path(), b"content-keyed").await; + assert!( + svc.get_cached_thumbnail(FILE_ID, None, SIZE, FMT, None) + .await + .is_none(), + "without a hash the content tiers are unreachable" + ); + + write_ext_sidecar(tmp.path(), b"per-file").await; + assert_eq!( + svc.get_cached_thumbnail(FILE_ID, None, SIZE, FMT, None) + .await + .as_deref(), + Some(&b"per-file"[..]), + "the per-file tier needs no hash and must still answer" + ); + } + + /// A timeout must never be recorded as a permanent verdict. + /// + /// It is the case that turns a load spike into permanent data loss: + /// `generate_and_persist` collapses every error into empty `Bytes`, which + /// is survivable only while that sentinel lives in moka and evicts. + /// Before any of it reaches `content_derived_blobs`, timeouts must + /// classify as transient — so this pins the mapping rather than trusting + /// the variant names to stay put. + #[test] + fn only_content_failures_are_permanent() { + // Facts about the image — safe to remember. + assert!(ThumbnailError::ImageError("decode failed".into()).is_permanent()); + assert!(ThumbnailError::UnsupportedFormat.is_permanent()); + + // Facts about the moment — must never be remembered. `timeout(...)` + // and the decode semaphore both surface as TaskError. + assert!(!ThumbnailError::TaskError("thumbnail generation timed out".into()).is_permanent()); + assert!(!ThumbnailError::TaskError("Decode semaphore closed".into()).is_permanent()); + assert!(!ThumbnailError::IoError("blob read failed".into()).is_permanent()); + } + + /// Empty bytes are moka's negative-entry convention (a previous render + /// failed). They must not be served as a thumbnail, or a failure gets + /// cached and returned as success. + #[tokio::test] + async fn empty_cache_entry_is_not_served() { + let tmp = tempfile::tempdir().unwrap(); + let svc = service(tmp.path()); + + svc.cache + .insert(ThumbnailCacheKey::content(HASH, SIZE, FMT), Bytes::new()) + .await; + write_blob_sidecar(tmp.path(), b"real-bytes").await; + + let got = svc + .get_cached_thumbnail(FILE_ID, Some(HASH), SIZE, FMT, None) + .await; + assert_eq!( + got.as_deref(), + Some(&b"real-bytes"[..]), + "a negative entry must fall through, not be served" + ); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/infrastructure/services/transcode_import_service.rs b/src/infrastructure/services/transcode_import_service.rs new file mode 100644 index 00000000..e1464c54 --- /dev/null +++ b/src/infrastructure/services/transcode_import_service.rs @@ -0,0 +1,584 @@ +//! Step 7 migration tenant: drains `.transcoded/` into the derived tier. +//! +//! The twin of `thumb_derived_import`, with one difference that shapes the +//! whole job: **the legacy tree is keyed by file, the destination by +//! content.** Thumbnail sidecars were already named by blob hash, so their +//! import was a move. These are named `{file_id}.webp`, so every entry has +//! to be re-keyed through `storage.files` before it can be stored. +//! +//! That re-keying is not bookkeeping — it is the point. A sandbox with five +//! `.skip` markers had three of them naming the same content, so the +//! file-keyed tree held three copies of one verdict. After the import that +//! is a single row, and any future upload of those bytes inherits it +//! instead of paying for the decision again. +//! +//! ### Two artifact kinds, one walk +//! +//! * `{file_id}.webp` — a cached transcode. Imported as a derived Blob. +//! * `{file_id}.webp.skip` — a zero-byte marker meaning "WebP came out +//! larger for this file". Imported as a NEGATIVE row (NULL `blob_hash`), +//! so the verdict survives the deletion of the directory holding it. +//! +//! Both are claimed by the same walk because they share a source file and +//! a cursor; splitting them would mean two passes over one directory and +//! two chances for the pair to disagree about what has been handled. +//! +//! ### What is deliberately not imported +//! +//! Entries whose file is gone. The destination row is keyed by content +//! hash, which is resolved *through* `storage.files` — no file, no hash, +//! nothing to key by. Under `repair` these are deleted, because they are +//! unimportable by definition and a run that keeps rediscovering them +//! never reports zero, so the gate for removing the directory never opens. + +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use sqlx::PgPool; +use tokio::fs; + +use crate::infrastructure::scheduler::{ + JobRegistry, JobRunArgs, JobStore, JobStoreProvider, Mutates, RecoverableJobHandler, + RunOutcome, RunStatus, record_or_log, +}; +use crate::infrastructure::services::dedup_service::DedupService; +use crate::infrastructure::services::thumb_derived_import_service::audit_sidecar_deleted; + +pub const TRANSCODE_IMPORT_JOB_NAME: &str = "transcode_import"; + +/// `content_derived_blobs.kind` for everything this job writes. Must match +/// `ImageTranscodeService::DERIVED_KIND`, or the import would file its rows +/// where the read path does not look. +const DERIVED_KIND: &str = "transcode"; + +/// The only variant this job handles. `.transcoded/` has exactly one +/// subdirectory today; a second output format would add a directory and a +/// variant together, and this walk would grow a loop rather than a branch. +const VARIANT_WEBP: &str = "webp"; + +/// Files handled between checkpoints. Each is a read plus, at most, a blob +/// write — deliberately smaller than a pure-DB sweep's page. +const BATCH_SIZE: usize = 100; + +pub struct TranscodeImport { + /// `{storage_path}/.transcoded`, matching `ImageTranscodeService::new`. + transcoded_root: PathBuf, + dedup: Arc, + /// Needed for the re-keying: file id → content hash. The thumbnail + /// imports have no equivalent because their sidecars were already + /// content-named. + pool: Arc, +} + +impl TranscodeImport { + pub fn new(transcoded_root: PathBuf, dedup: Arc, pool: Arc) -> Self { + Self { + transcoded_root, + dedup, + pool, + } + } + + pub async fn register_recoverable_job( + self: Arc, + registry: &JobRegistry, + provider: &Arc, + ) -> Arc { + // On-demand, matching the thumbnail imports. The boot run in repair + // mode IS the migration: nothing writes a hash-keyed entry here any + // more, so the tail cannot grow after startup, and a periodic tick + // could not finish the job anyway because ticks never pass + // `repair`. Once drained it would be a `read_dir` returning + // nothing, every day, forever. + registry + .register_recoverable_job(self.clone(), provider.clone(), None) + .await; + self + } + + /// The `webp/` subdirectory, where both artifact kinds live. + fn variant_dir(&self) -> PathBuf { + self.transcoded_root.join(VARIANT_WEBP) + } + + /// Sorted entry names, so the cursor totally orders the traversal. + /// + /// Returns both `{id}.webp` and `{id}.webp.skip`; the caller decides + /// which is which. Anything else is ignored rather than reported — the + /// directory is a local cache and has never promised to hold only our + /// files. + async fn entry_names(dir: &std::path::Path) -> Vec { + let Ok(mut entries) = fs::read_dir(dir).await else { + return Vec::new(); + }; + let mut names = Vec::new(); + while let Ok(Some(entry)) = entries.next_entry().await { + if let Some(name) = entry.file_name().to_str() + && parse_entry(name).is_some() + { + names.push(name.to_string()); + } + } + names.sort(); + names + } + + /// Resolve a file id to the BLAKE3 of its content. + /// + /// `None` means the file is gone — which is the unimportable case, not + /// an error: the destination is keyed by content, and a deleted file + /// has no content to key by. + async fn content_hash_of(&self, file_id: &str) -> Option { + sqlx::query_as::<_, (String,)>( + "SELECT blob_hash FROM storage.files WHERE id = $1::uuid AND blob_hash IS NOT NULL", + ) + .bind(file_id) + .fetch_optional(self.pool.as_ref()) + .await + .ok() + .flatten() + .map(|(h,)| h) + } +} + +/// Checkpoint once a batch has accumulated, resetting the counter. +/// +/// Returns `Some(RunOutcome::Failed)` when the store write fails, for the +/// caller to return. Shared by both exits of the loop body so the cursor +/// advances identically whether an entry was imported or skipped — the +/// alternative is two copies of this, which is how the first draft ended +/// up dropping a future and silently never checkpointing on one path. +async fn checkpoint_if_due( + store: &dyn JobStore, + name: &str, + since_checkpoint: &mut usize, +) -> Option { + if *since_checkpoint < BATCH_SIZE { + return None; + } + let scanned = *since_checkpoint as u64; + *since_checkpoint = 0; + match store.checkpoint(name.as_bytes().to_vec(), scanned).await { + Ok(()) => None, + Err(e) => Some(RunOutcome::Failed { + message: format!("checkpoint: {e}"), + }), + } +} + +/// What a `.transcoded/webp/` entry names. +/// +/// Returns the file id and whether it is a negative marker. Anything not +/// matching either shape is not ours. +fn parse_entry(name: &str) -> Option<(&str, bool)> { + if let Some(id) = name.strip_suffix(".webp.skip") { + return (!id.is_empty()).then_some((id, true)); + } + if let Some(id) = name.strip_suffix(".webp") { + return (!id.is_empty()).then_some((id, false)); + } + None +} + +#[async_trait] +impl RecoverableJobHandler for TranscodeImport { + fn name(&self) -> &str { + TRANSCODE_IMPORT_JOB_NAME + } + + fn description(&self) -> &'static str { + "Migrates cached WebP transcodes out of the legacy .transcoded/ \ + directory into content-addressed blob storage, re-keying each one \ + from its file id to its content hash. Entries for identical \ + content collapse into a single row, so the same image cached under \ + several files stops being stored several times. Zero-byte .skip \ + markers become negative rows, preserving the verdict that a file \ + is not worth transcoding." + } + + fn mutates(&self) -> Mutates { + Mutates::Always + } + + fn repair_description(&self) -> Option<&'static str> { + Some( + "Also DELETES each cached transcode once its replacement has \ + been read back and compared byte for byte, and removes the \ + directory when empty. Entries whose file no longer exists are \ + deleted without a readback — they cannot be re-keyed and \ + nothing can reference them again. Irreversible, though a \ + transcode is a pure function of its source: anything deleted \ + in error is recomputed on the next request.", + ) + } + + async fn count_total(&self) -> Option { + Some(Self::entry_names(&self.variant_dir()).await.len() as u64) + } + + async fn run_resumable( + &self, + store: &dyn JobStore, + args: &JobRunArgs, + resume_cursor: Option>, + ) -> RunOutcome { + // Cursor is the entry name. One directory, sorted, so the name + // alone totally orders the walk — unlike the thumbnail imports, + // which need `{size}/{name}` to span three directories. + let cursor: Option = match resume_cursor { + None => None, + Some(b) if b.is_empty() => None, + Some(b) => match String::from_utf8(b) { + Ok(s) => Some(s), + Err(e) => { + return RunOutcome::Failed { + message: format!("invalid cursor: not valid UTF-8: {e}"), + }; + } + }, + }; + + let delete_imported = args.repair; + let dir = self.variant_dir(); + + let mut imported = 0u64; + let mut negatives = 0u64; + let mut already = 0u64; + let mut file_gone = 0u64; + let mut deleted = 0u64; + let mut unverified = 0u64; + let mut failed = 0u64; + let mut since_checkpoint = 0usize; + // Last entry visited, for the tail flush below. + let mut last_name: Option = None; + + for name in Self::entry_names(&dir).await { + if let Some(c) = &cursor + && name.as_str() <= c.as_str() + { + continue; + } + + match store.status().await { + Ok(RunStatus::CancelRequested) => { + return RunOutcome::Paused { + cursor: name.into_bytes(), + }; + } + Ok(_) => {} + Err(e) => { + return RunOutcome::Failed { + message: format!("status poll: {e}"), + }; + } + } + + let Some((file_id, is_negative)) = parse_entry(&name) else { + continue; + }; + let path = dir.join(&name); + + // Re-key. This is the step the thumbnail imports do not have. + let Some(source_hash) = self.content_hash_of(file_id).await else { + file_gone += 1; + let mut removed = false; + if delete_imported && fs::remove_file(&path).await.is_ok() { + deleted += 1; + removed = true; + audit_sidecar_deleted( + TRANSCODE_IMPORT_JOB_NAME, + "file_gone", + file_id, + "-", + &path, + ); + } + record_or_log( + store, + TRANSCODE_IMPORT_JOB_NAME, + "transcode_file_gone", + "anomaly", + None, + serde_json::json!({ + "path": name, + "file_id": file_id, + "deleted": removed, + "note": "no storage.files row, so the entry cannot be re-keyed to a \ + content hash; unimportable and unreachable", + }), + ) + .await; + // Falls through to the shared checkpoint at the end of the + // loop rather than duplicating it here. An earlier draft + // did duplicate it and dropped the future without + // awaiting — the entry counted toward the batch, the + // cursor never advanced, and a resumed run would have + // rewalked everything already handled. + since_checkpoint += 1; + if let Some(failure) = checkpoint_if_due(store, &name, &mut since_checkpoint).await + { + return failure; + } + continue; + }; + + if is_negative { + // A verdict, not bytes. `store_derived_negative` is + // ON CONFLICT DO NOTHING, so the three markers that named + // one piece of content in the sandbox collapse here rather + // than fighting over the row. + match self + .dedup + .store_derived_negative(&source_hash, DERIVED_KIND, VARIANT_WEBP) + .await + { + Ok(()) => { + negatives += 1; + if delete_imported && fs::remove_file(&path).await.is_ok() { + deleted += 1; + audit_sidecar_deleted( + TRANSCODE_IMPORT_JOB_NAME, + "negative_imported", + file_id, + "-", + &path, + ); + } + } + Err(e) => { + failed += 1; + tracing::warn!( + target: "oxicloud::dedup", + event = "transcode_import.negative_failed", + file_id = %file_id, + source_hash = %source_hash, + error = %e, + "failed to record negative transcode row; entry kept" + ); + } + } + } else if self + .dedup + .find_derived_blob(&source_hash, DERIVED_KIND, VARIANT_WEBP) + .await + .is_some() + { + // Already imported — by an earlier run, or by another file + // sharing this content. Checked BEFORE storing so a re-run + // does not release and retake the reference. + already += 1; + if delete_imported { + let existing = self + .dedup + .find_derived_blob(&source_hash, DERIVED_KIND, VARIANT_WEBP) + .await; + if let Some(r) = existing { + if crate::infrastructure::services::thumb_derived_import_service::ThumbDerivedImport::verify_and_unlink( + &self.dedup, + TRANSCODE_IMPORT_JOB_NAME, + &source_hash, + &r.blob_hash, + &path, + ) + .await + { + deleted += 1; + } else { + unverified += 1; + record_or_log( + store, + TRANSCODE_IMPORT_JOB_NAME, + "transcode_delete_unverified", + "anomaly", + None, + serde_json::json!({ + "path": name, + "file_id": file_id, + "source_hash": source_hash, + "note": "stored transcode did not read back identical; \ + cached copy kept", + }), + ) + .await; + } + } + } + } else { + match fs::read(&path).await { + Ok(data) => { + match self + .dedup + .store_derived_blob( + &source_hash, + DERIVED_KIND, + VARIANT_WEBP, + "image/webp", + Bytes::from(data), + ) + .await + { + Ok(stored_hash) => { + imported += 1; + if delete_imported { + if crate::infrastructure::services::thumb_derived_import_service::ThumbDerivedImport::verify_and_unlink( + &self.dedup, + TRANSCODE_IMPORT_JOB_NAME, + &source_hash, + &stored_hash, + &path, + ) + .await + { + deleted += 1; + } else { + unverified += 1; + } + } + } + Err(e) => { + failed += 1; + tracing::warn!( + target: "oxicloud::dedup", + event = "transcode_import.store_failed", + file_id = %file_id, + source_hash = %source_hash, + error = %e, + "failed to store derived transcode; entry kept" + ); + } + } + } + Err(e) => { + failed += 1; + tracing::warn!( + target: "oxicloud::dedup", + event = "transcode_import.read_failed", + path = %path.display(), + error = %e, + "failed to read cached transcode; entry kept" + ); + } + } + } + + since_checkpoint += 1; + last_name = Some(name.clone()); + if let Some(failure) = checkpoint_if_due(store, &name, &mut since_checkpoint).await { + return failure; + } + } + + // Flush the tail. + // + // `checkpoint_if_due` only fires on a full batch, so a run shorter + // than BATCH_SIZE never checkpointed at all and reported + // `scanned_count: 0` against a known `total_rows` — the admin + // progress bar sat at zero through the whole run and finished + // there. Longer runs were wrong too, just less visibly: the + // remainder after the last full batch was never counted. + // + // Cursor-wise this is a no-op — the walk is finished, so nothing + // will resume from it — but the scanned delta is what the progress + // display reads, and it has to include the last partial batch. + if since_checkpoint > 0 + && let Some(name) = last_name + && let Err(e) = store + .checkpoint(name.into_bytes(), since_checkpoint as u64) + .await + { + return RunOutcome::Failed { + message: format!("final checkpoint: {e}"), + }; + } + + // Remove the tree once drained. Deletion first, rename only if a + // non-cache file is in the way — same rule as `.thumbnails/`, and + // for the same reason: absence is what the read path tests, and a + // stray `.DS_Store` must not keep the fallback alive forever. + if delete_imported { + let _ = fs::remove_dir(&dir).await; + match fs::remove_dir(&self.transcoded_root).await { + Ok(()) => tracing::info!( + target: "oxicloud::dedup", + event = "transcode_import.root_removed", + run_id = %store.run_id(), + path = %self.transcoded_root.display(), + "🧹 legacy transcode directory removed" + ), + Err(_) => { + let parked = self.transcoded_root.with_file_name(".transcoded.migrated"); + match fs::rename(&self.transcoded_root, &parked).await { + Ok(()) => tracing::info!( + target: "oxicloud::dedup", + event = "transcode_import.root_parked", + run_id = %store.run_id(), + to = %parked.display(), + "🧹 legacy transcode directory could not be removed (a non-cache \ + file remains) — moved aside instead" + ), + Err(e) => tracing::warn!( + target: "oxicloud::dedup", + event = "transcode_import.root_kept", + run_id = %store.run_id(), + reason = %e, + "legacy transcode directory neither removed nor moved aside" + ), + } + } + } + } + + tracing::info!( + target: "oxicloud::dedup", + event = "transcode_import.completed", + run_id = %store.run_id(), + imported = imported, + negatives = negatives, + already_present = already, + file_gone = file_gone, + deleted = deleted, + unverified = unverified, + failed = failed, + "transcode_import: {imported} imported, {negatives} negative verdict(s), \ + {already} already present, {file_gone} unimportable, {deleted} deleted, \ + {unverified} kept unverified, {failed} failed" + ); + + RunOutcome::completed_with(serde_json::json!({ + "imported": imported, + "negatives": negatives, + "already_present": already, + "file_gone": file_gone, + "deleted": deleted, + "unverified": unverified, + "failed": failed, + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The walk claims both artifact kinds and nothing else. + /// + /// `.webp.skip` must be tested BEFORE `.webp`, or the shorter suffix + /// matches first and every marker imports as if it were a cached + /// transcode — reading a zero-byte file and storing it as the + /// transcode of its source, which would then be served to clients. + #[test] + fn entry_names_are_parsed_by_longest_suffix_first() { + let id = "3f2b1c00-1111-2222-3333-444455556666"; + + assert_eq!(parse_entry(&format!("{id}.webp")), Some((id, false))); + assert_eq!(parse_entry(&format!("{id}.webp.skip")), Some((id, true))); + + // Not ours: no id, wrong extension, or a bare marker. + assert_eq!(parse_entry(".webp"), None); + assert_eq!(parse_entry(".webp.skip"), None); + assert_eq!(parse_entry(&format!("{id}.jpg")), None); + assert_eq!(parse_entry(".DS_Store"), None); + } +} diff --git a/src/infrastructure/services/trash_cleanup_service.rs b/src/infrastructure/services/trash_cleanup_service.rs index 8fa291b9..53437be9 100644 --- a/src/infrastructure/services/trash_cleanup_service.rs +++ b/src/infrastructure/services/trash_cleanup_service.rs @@ -6,7 +6,7 @@ use tracing::{debug, error, info, instrument}; use crate::common::errors::Result; use crate::domain::repositories::trash_repository::TrashRepository; use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository; -use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs}; +use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs, Mutates}; use crate::infrastructure::services::dedup_service::DedupService; use async_trait::async_trait; @@ -177,6 +177,17 @@ impl JobHandler for TrashCleanupService { Self::JOB_NAME } + fn description(&self) -> &'static str { + "Permanently deletes trashed items past the retention window, then \ + runs a dedup GC sweep as its tail step to reclaim blobs the \ + deletions dropped to zero references. This is the periodic tick \ + that keeps storage bounded." + } + + fn mutates(&self) -> Mutates { + Mutates::Always + } + /// Runs one bulk-delete-expired + GC sweep. `count` on the returned /// `JobOutcome::Ok` is the total number of rows this tick removed /// from the trash (files + folders); `extra` carries GC reclaim diff --git a/src/interfaces/api/handlers/admin_handler.rs b/src/interfaces/api/handlers/admin_handler.rs index 0cb66c26..455f672f 100644 --- a/src/interfaces/api/handlers/admin_handler.rs +++ b/src/interfaces/api/handlers/admin_handler.rs @@ -22,7 +22,7 @@ use crate::application::dtos::settings_dto::{ TestOidcConnectionDto, TestStorageConnectionDto, UpdateUserActiveDto, UpdateUserQuotaDto, UpdateUserRoleDto, }; -use crate::application::dtos::user_dto::{AdminUserSummaryDto, UserDto}; +use crate::application::dtos::user_dto::{FullUserDto, PublicUserDto}; use crate::application::ports::authorization_ports::AuthorizationEngine; use crate::application::ports::plugin_ports::{LogQuery, PluginManagementPort, PluginMgmtError}; // JobStoreProvider is used only by the storage-migration shims below, @@ -39,16 +39,14 @@ use crate::interfaces::middleware::auth::AuthUser; use std::sync::Arc; use uuid::Uuid; -#[derive(serde::Serialize)] -#[serde(untagged)] -enum AdminUsersPayload { - Full(Vec), - Summary(Vec), -} - +/// Response envelope for `GET /api/admin/users`. `users` is always +/// `Vec` — same shape one row of `/me`'s embedded +/// `full` block carries; the FE seeds `resolveUser` cache from +/// `row.user` (kills the per-row `/api/users/{id}` fetch). See +/// `docs/plan/userdto-refactor.md`. #[derive(serde::Serialize)] struct AdminUsersPageResponse { - users: AdminUsersPayload, + users: Vec, total: i64, limit: i64, offset: i64, @@ -161,6 +159,11 @@ pub fn admin_routes() -> Router> { // `/blob/{hash}`) stay at `/api/dedup/*`. .route("/dedup/stats", get(get_stats)) .route("/dedup/recalculate", post(recalculate_stats)) + // Transcode effectiveness. Nothing exposed these before, so there + // was no way to tell a served-from-cache response from one that + // re-ran the decode + encode — not from the outside, and not from + // a test either. + .route("/transcode/stats", get(get_transcode_stats)) // SMTP diagnostics .route("/smtp/info", get(get_smtp_info)) .route("/smtp/test", post(send_smtp_test)) @@ -931,6 +934,51 @@ pub async fn get_dashboard_stats( .await .map_err(|e| AppError::internal_error(format!("Database query failed: {}", e)))?; + // External account count — distinct query (not FILTERed into + // `stats_row` above) because `stats_row` scopes to + // `is_external = false` for the operational-seat counts. + // Externals form their own population; the dashboard renders them + // as a separate stat card in the "User accounts" section. + let external_users: i64 = + sqlx::query_scalar(r#"SELECT COUNT(*)::INT8 FROM auth.users WHERE is_external = true"#) + .fetch_one(db_pool.as_ref()) + .await + .map_err(|e| AppError::internal_error(format!("External user count failed: {}", e)))?; + + // Live-activity counts — projection over auth.sessions, same + // `ONLINE_WINDOW` (5 min) the Prometheus gauges use so the + // dashboard number, admin-table green dot, and + // `oxicloud_sessions_online` scrape all agree by construction. + // Bound as `$1 = window_secs` via `make_interval(secs => $1)` + // to keep the single-source-of-truth pattern (no SQL literal + // for the window). Both queries hit the partial index + // `idx_sessions_last_seen_at WHERE revoked = FALSE` so per-run + // cost is ~μs even at tens of thousands of session rows. + let online_window_secs: f64 = + crate::application::dtos::session_dto::ONLINE_WINDOW.as_secs_f64(); + let online_sessions: i64 = sqlx::query_scalar( + r#" + SELECT COUNT(*)::INT8 FROM auth.sessions + WHERE revoked = FALSE + AND last_seen_at > NOW() - make_interval(secs => $1) + "#, + ) + .bind(online_window_secs) + .fetch_one(db_pool.as_ref()) + .await + .map_err(|e| AppError::internal_error(format!("Online session count failed: {}", e)))?; + let online_users: i64 = sqlx::query_scalar( + r#" + SELECT COUNT(DISTINCT user_id)::INT8 FROM auth.sessions + WHERE revoked = FALSE + AND last_seen_at > NOW() - make_interval(secs => $1) + "#, + ) + .bind(online_window_secs) + .fetch_one(db_pool.as_ref()) + .await + .map_err(|e| AppError::internal_error(format!("Online user count failed: {}", e)))?; + use sqlx::Row; // Per-drive-kind quota panel: @@ -1015,6 +1063,9 @@ pub async fn get_dashboard_stats( total_users: stats_row.get("total_users"), active_users: stats_row.get("active_users"), admin_users: stats_row.get("admin_users"), + external_users, + online_users, + online_sessions, drive_usage, users_over_80_percent: stats_row.get("users_over_80"), users_over_quota: stats_row.get("users_over_quota"), @@ -1037,13 +1088,20 @@ pub async fn get_dashboard_stats( // ============================================================================ /// GET /api/admin/users?limit=50&offset=0 — list all users +/// +/// Always returns `Vec` — the shape one row of the +/// `/me` response's embedded `full` block carries. The former +/// `?summary` toggle (flat `PublicUserDto` vs nested `FullUserDto`) +/// has been retired: admin listing is low-volume and the FE always +/// asked for the nested shape anyway, so the two-shape split served +/// no caller and only invited jq-path bugs. See +/// `docs/plan/userdto-refactor.md`. #[utoipa::path( get, path = "/api/admin/users", params( ("limit" = Option, Query, description = "Max users to return (default 100, max 500)"), - ("offset" = Option, Query, description = "Pagination offset"), - ("summary" = Option, Query, description = "Return the compact management-table projection") + ("offset" = Option, Query, description = "Pagination offset") ), responses( (status = 200, description = "List of users"), @@ -1071,31 +1129,16 @@ pub async fn list_users( // internal-only variant is used by system address book / sharee // search, where surfacing externals would leak identities. See // `auth_application_service::list_users` doc for the split. - let users = if query.summary.unwrap_or(false) { - AdminUsersPayload::Summary( - auth.auth_application_service - .list_user_summaries_including_external_with_perms( - state.authorization.as_ref(), - auth_user.id, - limit, - offset, - ) - .await - .map_err(AppError::from)?, + let users = auth + .auth_application_service + .list_user_summaries_including_external_with_perms( + state.authorization.as_ref(), + auth_user.id, + limit, + offset, ) - } else { - AdminUsersPayload::Full( - auth.auth_application_service - .list_users_including_external_with_perms( - state.authorization.as_ref(), - auth_user.id, - limit, - offset, - ) - .await - .map_err(AppError::from)?, - ) - }; + .await + .map_err(AppError::from)?; let total = auth .auth_application_service @@ -1562,7 +1605,7 @@ pub async fn reset_user_password( path = "/api/admin/users/{id}/promote-to-internal", params(("id" = String, Path, description = "Target user id")), responses( - (status = 200, description = "User promoted", body = UserDto), + (status = 200, description = "User promoted", body = PublicUserDto), (status = 400, description = "Magic-link login is disabled on this deployment"), (status = 401, description = "Unauthorized"), (status = 403, description = "Admin required (or target is OIDC-linked)"), @@ -2464,6 +2507,51 @@ pub async fn delete_drive_admin( /// /// Production endpoint, always on. Read-only, so no audit line — /// the standard admin-middleware auth check is enough. +/// `GET /api/admin/transcode/stats` — WebP transcode effectiveness. +/// +/// The four counters distinguish where a response came from, which is +/// otherwise invisible: `transcodes` is work actually done, while +/// `cache_hits` (in-memory, keyed by file id) and `disk_hits` (the +/// durable content-keyed tier, plus the legacy local cache) are work +/// avoided. A rising `transcodes` against a flat `disk_hits` means the +/// derived tier is not being consulted — which is exactly the +/// regression a migration can introduce silently. +/// +/// `bytes_saved` counts only successful transcodes; images the encoder +/// could not shrink contribute nothing to it and are remembered as +/// negative rows instead. +/// +/// Read-only, so no audit line — the admin middleware gate is enough. +#[utoipa::path( + get, + path = "/api/admin/transcode/stats", + responses( + (status = 200, description = "Transcode statistics"), + (status = 401, description = "Unauthorized"), + (status = 403, description = "Admin required"), + ), + security(("bearerAuth" = [])), + tag = "admin" +)] +pub async fn get_transcode_stats(State(state): State>) -> impl IntoResponse { + let s = state.core.image_transcode_service.get_stats().await; + ( + StatusCode::OK, + Json(serde_json::json!({ + "cache_hits": s.cache_hits, + "disk_hits": s.disk_hits, + "transcodes": s.transcodes, + "bytes_saved": s.bytes_saved, + "transcode_errors": s.transcode_errors, + // Decodes that produced something larger. Work done for no + // gain — the thing the stored negative verdict prevents + // repeating, and invisible before this counter existed. + "not_beneficial": s.not_beneficial, + })), + ) + .into_response() +} + #[utoipa::path( get, path = "/api/admin/jobs", @@ -2524,6 +2612,28 @@ pub async fn list_jobs(State(state): State>) -> impl IntoResponse } } + // Mark the jobs `OXICLOUD_STARTUP_JOBS` dispatches at boot. Without + // this the panel is silently wrong about the most consequential thing + // on the row: a job configured with `repair=true` deletes files on + // every restart, and the row would suggest that only ever happens + // when someone clicks Run. + for job in summary.iter_mut() { + if let Some(configured) = state + .core + .config + .startup_jobs + .iter() + .find(|s| s.name == job.name) + { + job.startup = Some(crate::infrastructure::scheduler::StartupTrigger { + force: configured.args.force, + deep: configured.args.deep, + repair: configured.args.repair, + storage: configured.args.storage.clone(), + }); + } + } + (StatusCode::OK, Json(summary)).into_response() } @@ -2537,6 +2647,12 @@ pub async fn list_jobs(State(state): State>) -> impl IntoResponse /// `deep=true` opts into slow variants — `consistency_batch` fans it /// out to sub-jobs; `storage_consistency` (when implemented) will /// re-BLAKE3 each blob for bitrot detection. See `JobRunArgs.deep`. +/// +/// `repair=true` opts into corrective action on the refcount +/// consistency tenants (`blobs_consistency`, `manifests_consistency`, +/// and `consistency_batch` which fans out to both). Default `false` +/// preserves discovery-only. See `JobRunArgs.repair` for the +/// content-safety and race-safety guarantees. #[derive(serde::Deserialize)] pub struct TriggerJobQuery { #[serde(default)] @@ -2553,6 +2669,8 @@ pub struct TriggerJobQuery { /// `AppConfig.storage_entries`. #[serde(default)] pub storage: Option, + #[serde(default)] + pub repair: bool, } /// `POST /api/admin/jobs/{name}/trigger` — dispatch one run off-schedule. @@ -2591,15 +2709,18 @@ pub async fn trigger_job( job = %name, force = query.force, deep = query.deep, - "👮🏻‍♂️ Admin triggered job {} (force={}, deep={})", + repair = query.repair, + "👮🏻‍♂️ Admin triggered job {} (force={}, deep={}, repair={})", name, query.force, query.deep, + query.repair, ); let args = JobRunArgs { force: query.force, deep: query.deep, storage: query.storage.clone(), + repair: query.repair, }; // Jobs that can run for hours (backend_migration, future diff --git a/src/interfaces/api/handlers/app_password_handler.rs b/src/interfaces/api/handlers/app_password_handler.rs index f2b36b14..ed8a9023 100644 --- a/src/interfaces/api/handlers/app_password_handler.rs +++ b/src/interfaces/api/handlers/app_password_handler.rs @@ -61,7 +61,7 @@ async fn create_app_password( } // Require a claimed username. NextCloud Basic Auth resolves users by - // username; an app password is unusable without one. UserDto carries + // username; an app password is unusable without one. PublicUserDto carries // an empty string when the underlying `users.username` is NULL — the // entity rejects empty strings on construction, so empty here is an // unambiguous signal that the column is NULL. diff --git a/src/interfaces/api/handlers/auth_handler.rs b/src/interfaces/api/handlers/auth_handler.rs index 2f9bad63..236721b4 100644 --- a/src/interfaces/api/handlers/auth_handler.rs +++ b/src/interfaces/api/handlers/auth_handler.rs @@ -12,8 +12,8 @@ use uuid::Uuid; use crate::application::dtos::user_dto::{ AuthResponseDto, ChangePasswordDto, LoginDto, OidcCallbackQueryDto, OidcExchangeDto, - OidcProviderInfoDto, RefreshTokenDto, RegisterDto, SetupAdminDto, UpgradeToInternalDto, - UserDto, + OidcProviderInfoDto, PublicUserDto, RefreshTokenDto, RegisterDto, SelfUserDto, SetupAdminDto, + UpgradeToInternalDto, }; use crate::application::services::auth_application_service::{OidcCallbackResult, RegisterResult}; use crate::common::di::AppState; @@ -89,7 +89,7 @@ pub fn setup_route() -> Router> { /// the `audit` channel as `auth.register` with `reason` one of /// `created`, `email_taken`, `username_taken`. /// - **SMTP not configured**: there is no welcome-mail cover story, so -/// the classic `201 + UserDto` on success and `409` on collision +/// the classic `201 + PublicUserDto` on success and `409` on collision /// apply. Anti-enumeration would just be misleading UX (telling the /// user to check an email that will never arrive). Email-only /// signup is **503** in this mode because the user would otherwise @@ -106,7 +106,7 @@ pub fn setup_route() -> Router> { request_body = RegisterDto, responses( (status = 200, description = "Uniform registration response (SMTP configured, anti-enumeration mode)"), - (status = 201, description = "User registered successfully (SMTP not configured)", body = UserDto), + (status = 201, description = "User registered successfully (SMTP not configured)", body = PublicUserDto), (status = 400, description = "Validation error (malformed request body)"), (status = 403, description = "Registration disabled (admin setting or OIDC-only mode)"), (status = 409, description = "Username or email already taken (SMTP not configured)"), @@ -280,7 +280,7 @@ pub async fn register( } Ok(resp) } else { - // Classic mode: clear 201 + UserDto so the frontend can + // Classic mode: clear 201 + PublicUserDto so the frontend can // log the user in directly with the password they just // submitted. Unbox the DTO for the JSON serialisation. Ok((StatusCode::CREATED, Json(*user)).into_response()) @@ -626,7 +626,7 @@ pub async fn refresh_token( get, path = "/api/auth/me", responses( - (status = 200, description = "Current user profile", body = UserDto), + (status = 200, description = "Current user profile", body = SelfUserDto), (status = 401, description = "Not authenticated"), ), security(("bearerAuth" = [])), @@ -652,37 +652,21 @@ pub async fn get_current_user( // Semantics (`docs/plan/drive.md` §7): `storage_used_bytes` is the SUM // of `used_bytes` across the user's personal drives only. Shared drives // never count against this envelope — collaborating in a team drive - // costs no personal bytes. The matching cap is - // `storage_quota_bytes` (admin-only mutation). - let mut user = auth_service + // costs no personal bytes. + // + // Delegate to the shared `build_self_user_dto_for_id` — same code + // path `PATCH /me/profile` and `POST /upgrade-to-internal` use so + // all three self endpoints ship byte-for-byte identical shapes. + // The DPoP-bound signal comes from the JWT `cnf.jkt` claim + // (surfaced by the auth middleware into `AuthUser.dpop_jkt`); + // when present the session that minted this JWT is bound and + // the SPA can skip a redundant `/dpop/bind` call. + let self_dto = auth_service .auth_application_service - .get_user_by_id(user_id) + .build_self_user_dto_for_id(user_id, auth_user.dpop_jkt.is_some()) .await?; - // Overlay the cached `force_password_change` flag (see UserFlags). - // `From` defaults to false; the SPA reads this field on - // startup to decide whether to enter mandatory change-password - // mode. Using the cached path (`get_user_flags` → `user_flags_cache`) - // avoids a second DB round-trip on this hot endpoint. - if let Ok(flags) = auth_service - .auth_application_service - .get_user_flags(user_id) - .await - { - user.force_password_change = flags.force_password_change; - } - - // Session-binding state — read from the JWT `cnf.jkt` claim - // (surfaced by the auth middleware into `CurrentUser.dpop_jkt`). - // Present ⇒ the session that minted this JWT was bound; absent ⇒ - // the session is unbound and the SPA should call `/dpop/bind` - // to attach the browser's keypair (OIDC / magic-link redirect - // flow). Skips an otherwise-redundant `POST /dpop/bind` on every - // page load which would return 409 `already_bound` and litter - // the audit stream. - user.is_dpop_bound = auth_user.dpop_jkt.is_some(); - - Ok((StatusCode::OK, Json(user))) + Ok((StatusCode::OK, Json(self_dto))) } /// DTO for updating the user's profile image. @@ -837,14 +821,16 @@ pub async fn change_password( /// self-registration policy. Refused with 403 /// `error_type = "RegistrationDomainNotAllowed"`. /// -/// Response: the updated `UserDto` (post-upgrade view — `is_external` -/// is false, `storage_quota_bytes` is set). +/// Response: the updated `SelfUserDto` (same shape as `GET /me`) so the SPA +/// absorbs the post-upgrade state — new `storage_quota_bytes`, +/// `is_external = false`, updated OPAQUE / auth capability flags — in one +/// round trip without a follow-up `/me` fetch. #[utoipa::path( post, path = "/api/auth/upgrade-to-internal", request_body = UpgradeToInternalDto, responses( - (status = 200, description = "Upgrade succeeded", body = UserDto), + (status = 200, description = "Upgrade succeeded — returns SelfUserDto (same shape as GET /me)", body = SelfUserDto), (status = 400, description = "Password missing / too short"), (status = 401, description = "Not authenticated"), (status = 403, description = "OIDC user, or domain not in allowlist"), @@ -855,9 +841,10 @@ pub async fn change_password( )] pub async fn upgrade_to_internal( State(state): State>, - CurrentUserId(user_id): CurrentUserId, + auth_user: AuthUser, Json(dto): Json, ) -> Result { + let user_id = auth_user.id; let auth_service = state .auth_service .as_ref() @@ -905,7 +892,11 @@ pub async fn upgrade_to_internal( } } - let updated = auth_service + // Apply the upgrade. Service returns the updated `PublicUserDto`; + // we discard it and rebuild the full self view via the shared + // `build_self_user_dto_for_id` helper so the wire shape matches + // `GET /me` and `PATCH /me/profile` byte-for-byte. + let _ = auth_service .auth_application_service .upgrade_to_internal(user_id, dto) .await @@ -924,7 +915,11 @@ pub async fn upgrade_to_internal( _ => AppError::from(err), })?; - Ok((StatusCode::OK, Json(updated))) + let self_dto = auth_service + .auth_application_service + .build_self_user_dto_for_id(user_id, auth_user.dpop_jkt.is_some()) + .await?; + Ok((StatusCode::OK, Json(self_dto))) } /// Update the caller's profile (PR 24). @@ -942,7 +937,7 @@ pub async fn upgrade_to_internal( path = "/api/auth/me/profile", request_body = crate::application::dtos::user_dto::UpdateProfileDto, responses( - (status = 200, description = "Updated profile (UserDto)", body = UserDto), + (status = 200, description = "Updated profile (SelfUserDto) — same shape as GET /me so the SPA sees the just-written state without a follow-up fetch", body = SelfUserDto), (status = 400, description = "Validation error (e.g. invalid handle format, empty given_name)"), (status = 401, description = "Not authenticated"), (status = 403, description = "OIDC-managed profile — edit at the IdP"), @@ -953,20 +948,39 @@ pub async fn upgrade_to_internal( )] pub async fn update_profile( State(state): State>, - CurrentUserId(user_id): CurrentUserId, + auth_user: AuthUser, Json(dto): Json, ) -> Result { + let user_id = auth_user.id; let auth_service = state .auth_service .as_ref() .ok_or_else(|| AppError::internal_error("Authentication service not configured"))?; - let updated = auth_service + // Apply the patch. The service returns the updated `PublicUserDto` + // internally; we discard it and re-fetch the full self view below + // so the response matches `GET /me`'s `SelfUserDto` shape. + // + // Why SelfUserDto instead of PublicUserDto: a self-write endpoint + // whose response mirrors GET /me lets the SPA update its session + // store in one round trip. Returning a slim PublicUserDto would + // force the SPA to follow up with GET /me anyway to observe the + // just-written `ui_preferences` / `notify_on_share` / etc — those + // fields live on SelfUserDto only, not on the public identity + // slice. Same shape for both endpoints avoids "quiet lie" reads + // where a client PATCHes and then reads a stale local value. + let _ = auth_service .auth_application_service .update_profile_with_perms(user_id, dto, &state.locale_registry) .await?; - Ok((StatusCode::OK, Json(updated))) + // Rebuild via the shared helper so the wire shape matches + // `GET /me` and `POST /upgrade-to-internal` byte-for-byte. + let self_dto = auth_service + .auth_application_service + .build_self_user_dto_for_id(user_id, auth_user.dpop_jkt.is_some()) + .await?; + Ok((StatusCode::OK, Json(self_dto))) } // TODO: add utoipa @@ -1226,7 +1240,7 @@ pub struct BackchannelLogoutForm { path = "/api/setup", request_body = SetupAdminDto, responses( - (status = 201, description = "First admin created and system initialized", body = UserDto), + (status = 201, description = "First admin created and system initialized", body = PublicUserDto), (status = 403, description = "System already initialized"), (status = 503, description = "Auth service not configured"), ), @@ -1820,10 +1834,12 @@ pub async fn oidc_exchange( tracing::info!( "OIDC token exchange successful for user: {}", auth_response + .user + .full .user .username .as_deref() - .unwrap_or(&auth_response.user.email) + .unwrap_or(&auth_response.user.full.user.email) ); // Set HttpOnly cookies for the browser diff --git a/src/interfaces/api/handlers/contacts_handler.rs b/src/interfaces/api/handlers/contacts_handler.rs index b1b495f9..22fb9f22 100644 --- a/src/interfaces/api/handlers/contacts_handler.rs +++ b/src/interfaces/api/handlers/contacts_handler.rs @@ -16,7 +16,7 @@ use crate::application::dtos::contact_dto::{ AddressDto, ContactDto, ContactGroupDto, CreateContactDto, CreateContactGroupDto, EmailDto, GroupMembershipDto, PhoneDto, UpdateContactDto, UpdateContactGroupDto, }; -use crate::application::dtos::user_dto::UserDto; +use crate::application::dtos::user_dto::PublicUserDto; use crate::application::ports::carddav_ports::{AddressBookUseCase, ContactUseCase}; use crate::application::services::auth_application_service::AuthApplicationService; use crate::application::services::contact_service::ContactService; @@ -185,14 +185,14 @@ fn if_match_passes(if_match: Option<&str>, stored_etag: &str) -> bool { } } -/// Map a `UserDto` to a `ContactDto` so OxiCloud users appear as contacts +/// Map a `PublicUserDto` to a `ContactDto` so OxiCloud users appear as contacts /// inside the virtual system address book. /// /// `given_name`/`family_name` come from OIDC standard claims at JIT /// provisioning (or NULL for password-only or pre-OIDC users). When /// they're present, prefer a "First Last" full name; otherwise fall /// back to the username (which is always present). -fn user_to_contact(user: UserDto) -> ContactDto { +fn user_to_contact(user: PublicUserDto) -> ContactDto { // Display fallback chain: given+family name → username → email. // Username is `Option` post PR 16; externals start with None. let full_name = match (user.given_name.as_deref(), user.family_name.as_deref()) { @@ -222,8 +222,19 @@ fn user_to_contact(user: UserDto) -> ContactDto { photo_url: user.image.clone(), birthday: None, anniversary: None, - created_at: user.created_at, - updated_at: user.updated_at, + // System-book contacts are VIRTUAL projections of the user + // directory — they have no independent creation history. Stamp + // both timestamps with `Utc::now()` so the ContactDto shape is + // satisfied; CardDAV clients ETag on the vCard content (see + // `etag` below, keyed on the stable user id), not on these + // wrapper timestamps. + // + // Previously read `user.created_at` / `user.updated_at` from the + // fat `UserDto`; those fields moved to `FullUserDto` under the + // three-layer refactor (docs/plan/userdto-refactor.md) and are + // not exposed on the slim `PublicUserDto` this function receives. + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), etag: user.id, } } diff --git a/src/interfaces/api/handlers/file_handler.rs b/src/interfaces/api/handlers/file_handler.rs index f3cc381e..7fcf2194 100644 --- a/src/interfaces/api/handlers/file_handler.rs +++ b/src/interfaces/api/handlers/file_handler.rs @@ -393,21 +393,57 @@ impl FileHandler { // THUMBNAILS // ═══════════════════════════════════════════════════════════════════════ + /// Cache policy for every thumbnail response. + /// + /// **`private`**, because a thumbnail is authorization-gated: the handler + /// runs a `Permission::Read` check before serving it. `public` let any + /// shared cache — a corporate proxy, a CDN — store one user's thumbnail + /// and hand it to another. `Vary: Accept` did not help, because it does + /// not vary on `Authorization`. + /// + /// **`no-cache`**, not `immutable`, because this URL is keyed by file id + /// and its bytes are mutable: uploading a preview, replacing the file's + /// content, or removing an attachment all change what it serves. + /// `immutable` promises the opposite, so a client that fetched once would + /// not revalidate — for a year, under the previous `max-age` — and would + /// never see a new preview. That also made the content-keyed ETag + /// unobservable in a browser: a correct validator is worthless if nothing + /// asks. + /// + /// `no-cache` still stores the body; it only requires revalidation before + /// reuse, which the ETag answers with a body-less 304. + /// + /// The cost is a conditional request per thumbnail per page load. Buying + /// that back needs a content-addressed URL, where `immutable` would be + /// honest — but the hash would then be in the URL of an authorized + /// resource, so it stays `private` regardless. Separate change; it + /// touches the SPA and the file DTO. + /// Shared with the NextCloud preview endpoint, which is gated the same + /// way and must not drift from this policy. + pub(crate) const THUMBNAIL_CACHE_CONTROL: &'static str = "private, no-cache"; + /// Get a thumbnail for a file (image or video). /// - /// **Cache-first**: if the thumbnail already exists in the moka in-memory - /// cache or on disk, serve it immediately — **zero DB queries**. The - /// ownership check was already performed when the thumbnail was first - /// generated (at upload) or uploaded (PUT by the owner). UUIDv4 file IDs - /// have 122 bits of entropy, making enumeration infeasible. + /// **Cache-first**: once past the hash lookup below, a thumbnail already + /// in the moka in-memory cache or on disk is served without further DB + /// work. The ownership check was already performed when the thumbnail + /// was first generated (at upload) or uploaded (PUT by the owner). + /// UUIDv4 file IDs have 122 bits of entropy, making enumeration + /// infeasible. /// - /// **ETag / 304**: responses carry an immutable ETag. If the browser - /// sends `If-None-Match` matching the ETag, we return 304 Not Modified - /// without touching cache or DB — pure header round-trip. + /// **ETag / 304**: the ETag names the **blob actually served** — an + /// uploaded preview's hash, else a derived thumbnail's, else the + /// source-keyed form (see `ThumbnailService::thumbnail_content_id`). So + /// replacing content or uploading a preview invalidates correctly, and + /// two files serving identical bytes share a validator. Costs one or two + /// indexed lookups on the 304 path, which an id-keyed ETag avoided at the + /// price of never invalidating. Cache policy is + /// [`Self::THUMBNAIL_CACHE_CONTROL`] — `private, no-cache`, since this + /// URL is authorization-gated and its bytes are mutable. /// - /// The DB path is only taken on a **cache miss for images** where the - /// thumbnail hasn't been generated yet (first access after upload if - /// background generation hasn't finished). + /// Beyond that, the DB path is only taken on a **cache miss for images** + /// where the thumbnail hasn't been generated yet (first access after + /// upload if background generation hasn't finished). pub(super) async fn get_thumbnail_impl( State(state): State, auth_user: AuthUser, @@ -449,23 +485,52 @@ impl FileHandler { let format = ThumbnailFormat::from_accept(headers.get(header::ACCEPT).and_then(|v| v.to_str().ok())); - // ── ETag short-circuit (Solution C) ────────────────────────── - // Thumbnails are immutable — the ETag never changes for a given - // (file_id, size, format) triple. If the browser already has it, return - // 304 with zero I/O or DB work. Format is in the ETag so a client that - // switched codecs doesn't get a stale 304. - let etag = { - let (s, f) = (thumb_size.as_str(), format.as_str()); - let mut e = String::with_capacity(9 + id.len() + s.len() + f.len()); - e.push_str("\"thumb-"); - e.push_str(&id); - e.push('-'); - e.push_str(s); - e.push('-'); - e.push_str(f); - e.push('"'); - e + // ── ETag short-circuit ─────────────────────────────────────── + // Keyed on the CONTENT served, not the file id. + // + // Keying on `file_id` was wrong in both directions. Replacing a + // file's content preserves its id (`file_upload_service` rebuilds the + // entity with `parts.id` and a new hash, then fires + // `on_file_updated`, which regenerates the thumbnails), so the ETag + // never changed — and the response was `immutable` with a one-year + // max-age, so clients never revalidated and kept the old preview. + // Conversely a copy, or any dedup twin, got a *different* id and so + // refetched bytes it already held, even though the server serves both + // from the same derived blob. + // + // Cost: one PK lookup, where the id-keyed version needed none. It + // buys correct invalidation plus 304s shared across every file with + // the same content. The lookup runs after the authz check above, + // which has already hit the database. + // + // No new disclosure: `content_hash` is already on `FileDto` and + // returned by `GET /api/files/{id}`, so any caller who reaches here + // could read it anyway. + let blob_hash = match state + .repositories + .file_read_repository + .get_blob_hash(&id) + .await + { + Ok(h) => h, + Err(err) => return AppError::from(err).into_response(), }; + // The identity of the bytes about to be served, resolved through the + // same tier precedence the read path uses — an uploaded preview's own + // hash, else a derived thumbnail's own hash, else the source-keyed + // form. See `ThumbnailService::thumbnail_content_id`. + let etag = format!( + "\"{}\"", + thumbnail_service + .thumbnail_content_id( + &id, + &blob_hash, + thumb_size.into(), + format, + Some(&state.core.dedup_service), + ) + .await + ); if let Some(if_none_match) = headers.get(header::IF_NONE_MATCH) && let Ok(val) = if_none_match.to_str() && (val == etag || val == "*") @@ -474,7 +539,7 @@ impl FileHandler { .status(StatusCode::NOT_MODIFIED) .header(header::ETAG, &etag) .header(header::VARY, header::ACCEPT.as_str()) - .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .header(header::CACHE_CONTROL, Self::THUMBNAIL_CACHE_CONTROL) .body(Body::empty()) .unwrap() .into_response(); @@ -484,7 +549,15 @@ impl FileHandler { // Try moka (RAM) → disk before touching the database. // If the thumbnail exists it was authorized at creation time. if let Some(data) = thumbnail_service - .get_cached_thumbnail(&id, None, thumb_size.into(), format) + .get_cached_thumbnail( + &id, + // Already resolved for the ETag above — hand it over rather + // than let the service look it up a second time. + Some(&blob_hash), + thumb_size.into(), + format, + Some(&state.core.dedup_service), + ) .await { return Response::builder() @@ -494,7 +567,7 @@ impl FileHandler { crate::common::mime_detect::thumbnail_content_type(&data), ) .header(header::CONTENT_LENGTH, data.len()) - .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .header(header::CACHE_CONTROL, Self::THUMBNAIL_CACHE_CONTROL) .header(header::ETAG, &etag) .header(header::VARY, header::ACCEPT.as_str()) .body(Body::from(data)) @@ -528,20 +601,15 @@ impl FileHandler { .into_response(); } - // Resolve the blob hash (content-addressable storage). - let blob_hash = match state - .repositories - .file_read_repository - .get_blob_hash(&id) - .await - { - Ok(hash) => hash, - Err(_) => { - return AppError::internal_error("File blob not found").into_response(); - } - }; + // `blob_hash` was resolved above to build the ETag — no second lookup. if let Some(data) = thumbnail_service - .get_cached_thumbnail(&id, Some(&blob_hash), thumb_size.into(), format) + .get_cached_thumbnail( + &id, + Some(&blob_hash), + thumb_size.into(), + format, + Some(&state.core.dedup_service), + ) .await { return Response::builder() @@ -551,7 +619,7 @@ impl FileHandler { crate::common::mime_detect::thumbnail_content_type(&data), ) .header(header::CONTENT_LENGTH, data.len()) - .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .header(header::CACHE_CONTROL, Self::THUMBNAIL_CACHE_CONTROL) .header(header::ETAG, &etag) .header(header::VARY, header::ACCEPT.as_str()) .body(Body::from(data)) @@ -572,6 +640,7 @@ impl FileHandler { Some(&blob_hash), thumb_size.into(), ThumbnailFormat::Webp, + Some(&state.core.dedup_service), ) .await { @@ -582,7 +651,7 @@ impl FileHandler { crate::common::mime_detect::thumbnail_content_type(&data), ) .header(header::CONTENT_LENGTH, data.len()) - .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .header(header::CACHE_CONTROL, Self::THUMBNAIL_CACHE_CONTROL) .header(header::ETAG, &etag) .header(header::VARY, header::ACCEPT.as_str()) .body(Body::from(data)) @@ -614,7 +683,7 @@ impl FileHandler { crate::common::mime_detect::thumbnail_content_type(&data), ) .header(header::CONTENT_LENGTH, data.len()) - .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .header(header::CACHE_CONTROL, Self::THUMBNAIL_CACHE_CONTROL) .header(header::ETAG, &etag) .header(header::VARY, header::ACCEPT.as_str()) .body(Body::from(data)) @@ -689,15 +758,71 @@ impl FileHandler { return AppError::from(err).into_response(); } - // Validate, re-encode to WebP, and store - match thumbnail_service + // Validate, re-encode, and store the per-file sidecar. + let stored = match thumbnail_service .store_external_thumbnail(&id, thumb_size.into(), body) .await { - Ok(_) => StatusCode::CREATED.into_response(), - Err(err) => AppError::internal_error(format!("Failed to store thumbnail: {}", err)) - .into_response(), + Ok(bytes) => bytes, + Err(err) => { + return AppError::internal_error(format!("Failed to store thumbnail: {}", err)) + .into_response(); + } + }; + + // Also record it as a file-keyed attachment. + // + // The sidecar above is `ext-{file_id}.jpg` on local disk, which no + // copy path duplicates and no other instance can see. Without this + // row a copied file loses the preview its owner uploaded — falling + // back to a rendered thumbnail, or to nothing at all for a PDF, which + // has no server-side render path. `copy_file_satellites` duplicates + // the row, so the copy inherits the bytes. + // + // File-keyed, never content-keyed: these bytes are the uploader's + // claim about THIS file, and sharing them across files with identical + // content is the poisoning vector `storage.file_attached_blobs` + // exists to prevent. + // + // Best-effort: the sidecar already succeeded, so the user has their + // thumbnail. Failing the request here would report an error for an + // operation that visibly worked. + if let Err(e) = state + .core + .dedup_service + .store_attached_blob( + &id, + "preview", + thumb_size.dir_name(), + "image/jpeg", + stored, + auth_user.id, + ) + .await + { + // FATAL as of step 10d2, where it used to warn and return 201. + // + // That was safe only while `ext-{file_id}.jpg` existed as a + // second copy. With the sidecar gone this is the ONLY durable + // home for bytes that have no server-side render path — a + // client-generated PDF preview cannot be recreated — so + // succeeding here would lose a user's upload behind a success + // response. Silent, and unrecoverable. + // + // The RAM entry is dropped too, or the cache would keep serving a + // preview that was never persisted and vanishes on eviction, + // contradicting the error the client just received. + let _ = thumbnail_service.delete_thumbnails(&id).await; + tracing::error!( + target: "oxicloud::dedup", + error = %e, + file_id = %id, + "failed to record attached thumbnail; upload rejected" + ); + return AppError::internal_error("Failed to store thumbnail").into_response(); } + + StatusCode::CREATED.into_response() } // ═══════════════════════════════════════════════════════════════════════ @@ -1437,7 +1562,7 @@ pub async fn list_files_query( #[utoipa::path( post, path = "/api/files/upload", - request_body(content_type = "multipart/form-data", description = "File data + optional folder_id field"), + request_body(content_type = "multipart/form-data", description = "File data + folder_id (required: it determines the file's owner and drive)"), responses( (status = 201, description = "File uploaded", body = FileDto), (status = 400, description = "Invalid request"), diff --git a/src/interfaces/api/handlers/magic_link_handler.rs b/src/interfaces/api/handlers/magic_link_handler.rs index aa2c07bf..8a376387 100644 --- a/src/interfaces/api/handlers/magic_link_handler.rs +++ b/src/interfaces/api/handlers/magic_link_handler.rs @@ -625,7 +625,7 @@ fn redirect_target(redemption: &MagicLinkRedemption) -> String { (Some(MagicLinkResourceKind::Folder), Some(folder_id)) => { format!("/files/{}", folder_id) } - _ if redemption.auth.user.is_external => "/shared-with-me".to_string(), + _ if redemption.auth.user.full.user.is_external => "/shared-with-me".to_string(), _ => "/files".to_string(), } } diff --git a/src/interfaces/api/handlers/users_handler.rs b/src/interfaces/api/handlers/users_handler.rs index b2d0d943..f6aee02e 100644 --- a/src/interfaces/api/handlers/users_handler.rs +++ b/src/interfaces/api/handlers/users_handler.rs @@ -1,6 +1,6 @@ //! User-profile lookup for the frontend. //! -//! `GET /api/users/{id}` returns a [`UserDto`] for the target user iff +//! `GET /api/users/{id}` returns a [`PublicUserDto`] for the target user iff //! the authenticated caller has a legitimate relationship with them. //! The visibility rule lives in //! [`AuthApplicationService::get_user_profile`] — handlers never embed diff --git a/src/interfaces/api/handlers/webdav_handler.rs b/src/interfaces/api/handlers/webdav_handler.rs index 3700ea6c..22fd0374 100644 --- a/src/interfaces/api/handlers/webdav_handler.rs +++ b/src/interfaces/api/handlers/webdav_handler.rs @@ -36,6 +36,7 @@ use crate::application::services::folder_service::FolderService; use crate::common::di::AppState; use crate::domain::repositories::drive_repository::DriveRepository; use crate::domain::services::authorization::{Permission, Resource, Subject}; +use crate::domain::services::path_service::normalize_storage_name; use crate::infrastructure::services::path_resolver_service::ResolvedResource; use crate::infrastructure::services::webdav_dead_property_store::{DeadPropertyStore, ResourceRef}; use crate::interfaces::errors::AppError; @@ -2452,6 +2453,18 @@ async fn handle_mkcol( )); } + // Capture the URL-path segments BEFORE scope-resolution rewrites `path` + // to `scope.db_path` — we need the original request URL to reconstruct + // the canonical `Content-Location` when the last segment gets NFC- + // normalized. The last segment is the same either way (it's the target + // resource name), but the URL prefix (including any `@drive/` + // routing tokens the client used) is only preserved here. + let request_url_segments: Vec = path + .split('/') + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .collect(); + // RFC 4918 §9.3.1: MKCOL on an existing URL MUST return 405. // RFC 4918 §9.3.1: MKCOL without an existing parent MUST return 409. // This handler only creates a single collection (the last path segment). @@ -2556,8 +2569,15 @@ async fn handle_mkcol( } }; + // NFC-normalize the client-supplied last segment so we can emit + // `Content-Location` when the canonical URL differs from what the + // client sent. The repo layer normalizes again on the way to the DB + // (idempotently — `is_nfc_quick` returns immediately for already-NFC + // input); doing it here too gives the handler a cheap way to know + // whether the URL changed. See AtalayaLabs/OxiCloud#706. + let normalized_segment = normalize_storage_name(new_segment); let create_dto = crate::application::dtos::folder_dto::CreateFolderDto { - name: new_segment.to_string(), + name: normalized_segment.clone(), parent_id, }; folder_service @@ -2565,10 +2585,57 @@ async fn handle_mkcol( .await .map_err(AppError::from)?; - Ok(Response::builder() - .status(StatusCode::CREATED) - .body(Body::empty()) - .unwrap()) + // If the client sent an NFD name (macOS Finder, some Android sync + // clients) and we canonicalised it, tell them the authoritative URL + // via `Content-Location` (RFC 7231 §3.1.4.2). Well-behaved clients + // (NextCloud desktop, rclone) update their local index; naive + // clients ignore the header (safely — status stays 201). Emitting + // only when the segment actually changed keeps the wire clean on + // the common ASCII / already-NFC path. + let mut response = Response::builder().status(StatusCode::CREATED); + if normalized_segment != new_segment { + response = response.header( + "Content-Location", + canonical_collection_url(&request_url_segments, &normalized_segment), + ); + } + Ok(response.body(Body::empty()).unwrap()) +} + +/// Reconstruct the canonical `Content-Location` value for a WebDAV +/// resource whose last URL segment was NFC-normalized server-side. +/// +/// Takes the original request-URL segments (as split by `/` after the +/// `/webdav/` prefix) and the canonical last-segment string, and +/// returns a full `/webdav/…/` URL with each segment individually +/// percent-encoded. Collection responses append a trailing `/` per +/// RFC 4918 §5.2. +fn canonical_collection_url(request_url_segments: &[String], canonical_last: &str) -> String { + let mut out = String::with_capacity( + request_url_segments.iter().map(|s| s.len()).sum::() + canonical_last.len() + 16, + ); + out.push_str("/webdav/"); + // Walk all segments except the last; the last is replaced with the + // canonical (normalized) form. + let prefix = if request_url_segments.len() > 1 { + &request_url_segments[..request_url_segments.len() - 1] + } else { + &[][..] + }; + for seg in prefix { + let _ = std::fmt::Write::write_fmt( + &mut out, + format_args!("{}/", utf8_percent_encode(seg, PATH_SEGMENT_ENCODE_SET)), + ); + } + let _ = std::fmt::Write::write_fmt( + &mut out, + format_args!( + "{}/", + utf8_percent_encode(canonical_last, PATH_SEGMENT_ENCODE_SET) + ), + ); + out } /** diff --git a/src/interfaces/api/mod.rs b/src/interfaces/api/mod.rs index 03d7008e..9bc4cfcc 100644 --- a/src/interfaces/api/mod.rs +++ b/src/interfaces/api/mod.rs @@ -46,7 +46,7 @@ use crate::application::dtos::trash_dto::{ }; use crate::application::dtos::user_dto::{ AuthResponseDto, ChangePasswordDto, LoginDto, OidcExchangeDto, OidcProviderInfoDto, - RefreshTokenDto, RegisterDto, SetupAdminDto, UserDto, + PublicUserDto, RefreshTokenDto, RegisterDto, SetupAdminDto, }; use crate::application::ports::chunked_upload_ports::{ ChunkUploadResponseDto, CreateUploadResponseDto, UploadStatusResponseDto, @@ -367,7 +367,7 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload; PaginationDto, PaginationRequestDto, // User / Auth schemas - UserDto, + PublicUserDto, LoginDto, RegisterDto, SetupAdminDto, @@ -583,7 +583,7 @@ mod tests { "FolderDto", "ShareDto", "TrashedItemDto", - "UserDto", + "PublicUserDto", ] { assert!(schemas.contains_key(name), "missing schema: {name}"); } diff --git a/src/interfaces/middleware/auth.rs b/src/interfaces/middleware/auth.rs index 7f2a70ec..a2f2f6a1 100644 --- a/src/interfaces/middleware/auth.rs +++ b/src/interfaces/middleware/auth.rs @@ -229,6 +229,18 @@ pub async fn auth_middleware( request.extensions_mut().insert(current_user); tracing::Span::current() .record("user_id", tracing::field::display(user_id)); + // Bump per-session liveness for the + // Prometheus gauges. O(1) DashMap upsert + // — no I/O on this hot path. The `sid` + // claim is `None` on tokens minted by + // pre-`sid` builds, in which case the + // stamp is skipped entirely — no + // fallback lookup, no round-trip. + if let (Some(sid), Some(tracker)) = + (claims.sid, state.last_seen_tracker.as_ref()) + { + tracker.stamp(sid); + } return Ok(next.run(request).await); } Err(e) => { @@ -353,6 +365,15 @@ pub async fn auth_middleware( request.extensions_mut().insert(CookieAuthenticated); tracing::Span::current() .record("user_id", tracing::field::display(user_id)); + // Cookie-auth branch stamps the same + // way as the Bearer branch above — + // see that site for the O(1) / + // no-DB rationale. + if let (Some(sid), Some(tracker)) = + (claims.sid, state.last_seen_tracker.as_ref()) + { + tracker.stamp(sid); + } return Ok(next.run(request).await); } LiveRole::Revoked => { diff --git a/src/interfaces/nextcloud/ocs_handler.rs b/src/interfaces/nextcloud/ocs_handler.rs index da1c19b1..39e832d8 100644 --- a/src/interfaces/nextcloud/ocs_handler.rs +++ b/src/interfaces/nextcloud/ocs_handler.rs @@ -191,7 +191,15 @@ async fn user_provisioning_response( return Json(ocs_err(997, "Database pool not available")).into_response(); }; - let user_dto = match auth_service + // Two-step lookup: (1) `get_user_profile_by_username_with_perms` + // gates access via the same visibility engine the REST endpoint + // uses; (2) if visibility passes, `get_user_with_derived_flags` + // hydrates the OCS-specific fields (federation_kind / last_login_at + // / active) that live on `FullUserDto` but not on the slim + // `PublicUserDto` returned by the visibility gate. Second call is + // ~1 DB round-trip on the maintenance pool; NC OCS provisioning is + // not on any hot inner loop. + let public = match auth_service .get_user_profile_by_username_with_perms( user.id, &userid, @@ -205,9 +213,27 @@ async fn user_provisioning_response( return Json(ocs_err(404, "User not found")).into_response(); } }; + let target_id = match uuid::Uuid::parse_str(&public.id) { + Ok(u) => u, + Err(_) => { + // Should be unreachable — PublicUserDto.id is always the + // serialised form of a Uuid. Fail closed if this invariant + // is ever violated. + return Json(ocs_err(500, "Malformed user id")).into_response(); + } + }; + let user_dto = match auth_service.get_user_with_derived_flags(target_id).await { + Ok((user, flags)) => crate::application::dtos::user_dto::FullUserDto::build(user, flags), + Err(_) => { + // Visibility already passed above; a miss here would mean + // the user was deleted between the two round-trips. Fall + // back to the 404 shape (anti-enum invariant still holds). + return Json(ocs_err(404, "User not found")).into_response(); + } + }; // Determine groups based on role - let groups = if user_dto.role == "admin" { + let groups = if user_dto.user.role == "admin" { vec!["admin", "users"] } else { vec!["users"] @@ -235,7 +261,7 @@ async fn user_provisioning_response( // Fetch quota from storage usage service let quota: (i64, i64) = match state.storage_usage_service.as_ref() { Some(service) => match service - .get_user_storage_info(uuid::Uuid::parse_str(&user_dto.id).unwrap_or_default()) + .get_user_storage_info(uuid::Uuid::parse_str(&user_dto.user.id).unwrap_or_default()) .await { Ok((used, total)) => (used, total), @@ -256,10 +282,10 @@ async fn user_provisioning_response( "meta": { "status": "ok", "statuscode": statuscode, "message": "OK" }, "data": { "enabled": user_dto.active, - "id": user_dto.username, - "display-name": user_dto.username, - "displayname": user_dto.username, - "email": user_dto.email, + "id": user_dto.user.username, + "display-name": user_dto.user.username, + "displayname": user_dto.user.username, + "email": user_dto.user.email, "phone": "", "address": "", "website": "", diff --git a/src/interfaces/nextcloud/preview_handler.rs b/src/interfaces/nextcloud/preview_handler.rs index c137112b..6f8e1c0b 100644 --- a/src/interfaces/nextcloud/preview_handler.rs +++ b/src/interfaces/nextcloud/preview_handler.rs @@ -17,6 +17,9 @@ use crate::application::ports::storage_ports::FileReadPort; use crate::application::ports::thumbnail_ports::{ThumbnailFormat, ThumbnailPort, ThumbnailSize}; use crate::common::di::AppState; use crate::domain::services::authorization::{Permission, Resource, Subject}; +// One definition of the thumbnail cache policy, shared with the REST +// endpoint: both are Permission::Read gated, so both must stay `private`. +use crate::interfaces::api::handlers::file_handler::FileHandler; use crate::interfaces::middleware::auth::AuthUser; use uuid::Uuid; @@ -137,31 +140,60 @@ pub async fn handle_preview( } }; - // Conditional revalidation — the ETag is derived from (object id, size) - // only, so it is computable right here, BEFORE the blob-hash query and - // the thumbnail cache/disk read. NC clients revalidate gallery previews - // constantly; the REST thumbnail endpoint has honoured `If-None-Match` - // since PHOTOS-ETAG — this endpoint set an immutable ETag but never - // compared it, so every revalidation re-ran the whole pipeline and - // re-shipped the body (ROUND10). Authz already passed above; a 304 - // must never skip the Read check. - let etag = { - let s = thumb_size.as_str(); - let mut e = String::with_capacity(9 + object_id.len() + s.len()); - e.push_str("\"thumb-"); - e.push_str(&object_id); - e.push('-'); - e.push_str(s); - e.push('"'); - e + // Conditional revalidation. NC clients revalidate gallery previews + // constantly; this endpoint set an immutable ETag but never compared it, + // so every revalidation re-ran the whole pipeline and re-shipped the body + // (ROUND10). Authz already passed above; a 304 must never skip the Read + // check. + // + // Keyed on the CONTENT of the bytes served, matching the REST thumbnail + // endpoint. Keying on the object id meant replacing a file's content — + // which preserves the id — left the validator unchanged, and the response + // was `immutable` with a one-year max-age, so clients never revalidated + // and showed the old preview indefinitely. Both halves are fixed: the + // ETag names what is served (see `thumbnail_content_id`) and the policy + // is `private, no-cache` (see `FileHandler::THUMBNAIL_CACHE_CONTROL`). + // + // This moves the blob-hash query ahead of the 304 rather than adding one: + // the same lookup used to sit just below, on the path that renders. + let blob_hash = match state + .repositories + .file_read_repository + .get_blob_hash(&object_id) + .await + { + Ok(hash) => hash, + Err(_) => { + return Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("File blob not found")) + .unwrap(); + } }; + // Same tier-precedence resolution as the REST endpoint: an uploaded + // preview's own hash, else a derived thumbnail's own hash, else the + // source-keyed form. NC pins JPEG, so that is the format asked for. + let etag = format!( + "\"{}\"", + state + .core + .thumbnail_service + .thumbnail_content_id( + &object_id, + &blob_hash, + thumb_size.into(), + ThumbnailFormat::Jpeg, + Some(&state.core.dedup_service), + ) + .await + ); if let Some(inm) = req.headers().get(header::IF_NONE_MATCH) && let Ok(client_etag) = inm.to_str() && (client_etag == etag || client_etag == "*") { return Response::builder() .status(StatusCode::NOT_MODIFIED) - .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .header(header::CACHE_CONTROL, FileHandler::THUMBNAIL_CACHE_CONTROL) .header(header::ETAG, etag) .body(Body::empty()) .unwrap(); @@ -179,21 +211,7 @@ pub async fn handle_preview( .unwrap(); } - // Resolve the blob hash (content-addressable storage) - let blob_hash = match state - .repositories - .file_read_repository - .get_blob_hash(&object_id) - .await - { - Ok(hash) => hash, - Err(_) => { - return Response::builder() - .status(StatusCode::NOT_FOUND) - .body(Body::from("File blob not found")) - .unwrap(); - } - }; + // `blob_hash` was resolved above to build the ETag. if let Some(data) = state .core .thumbnail_service @@ -204,6 +222,7 @@ pub async fn handle_preview( Some(&blob_hash), thumb_size.into(), ThumbnailFormat::Jpeg, + Some(&state.core.dedup_service), ) .await { @@ -211,7 +230,7 @@ pub async fn handle_preview( .status(StatusCode::OK) .header(header::CONTENT_TYPE, "image/jpeg") .header(header::CONTENT_LENGTH, data.len()) - .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .header(header::CACHE_CONTROL, FileHandler::THUMBNAIL_CACHE_CONTROL) .header(header::ETAG, etag) .body(Body::from(data)) .unwrap(); @@ -236,7 +255,7 @@ pub async fn handle_preview( .status(StatusCode::OK) .header(header::CONTENT_TYPE, "image/jpeg") .header(header::CONTENT_LENGTH, data.len()) - .header(header::CACHE_CONTROL, "public, max-age=31536000, immutable") + .header(header::CACHE_CONTROL, FileHandler::THUMBNAIL_CACHE_CONTROL) .header(header::ETAG, etag) .body(Body::from(data)) .unwrap(), diff --git a/src/interfaces/nextcloud/webdav_handler.rs b/src/interfaces/nextcloud/webdav_handler.rs index 0185ac2d..769cf179 100644 --- a/src/interfaces/nextcloud/webdav_handler.rs +++ b/src/interfaces/nextcloud/webdav_handler.rs @@ -27,6 +27,7 @@ use crate::application::ports::trash_ports::TrashUseCase; use crate::common::di::AppState; use crate::common::mime_detect::filename_from_path; use crate::domain::services::authorization::{Permission, Resource, Subject}; +use crate::domain::services::path_service::normalize_storage_name; use crate::infrastructure::services::path_resolver_service::ResolvedResource; use crate::infrastructure::services::webdav_dead_property_store::ResourceRef; use crate::interfaces::api::handlers::webdav_handler::{ @@ -1295,6 +1296,13 @@ async fn handle_mkcol( } let (target_name, parent_segments) = segments.split_last().expect("checked non-empty above"); + // NFC-normalize the client-supplied last segment so we can emit + // `Content-Location` if the canonical URL differs. Repo also + // normalizes (idempotent — `is_nfc_quick` fast path). See + // AtalayaLabs/OxiCloud#706 for the class of bug this closes on the + // NC surface (macOS Finder / NC desktop client emit NFD on macOS). + let normalized_target = normalize_storage_name(target_name); + // Take POC's `chroot`-based root resolution (drive-aware mount // point) but keep HEAD's parent_path lookup pattern — the // continuation below uses `get_folder_by_path(&parent_path, @@ -1320,7 +1328,7 @@ async fn handle_mkcol( }; let dto = CreateFolderDto { - name: target_name.to_string(), + name: normalized_target.clone(), parent_id: Some(parent_folder.id.clone()), }; // AuthZ audit #7 (2026-07-12): route `_with_perms` errors through @@ -1332,10 +1340,25 @@ async fn handle_mkcol( .await .map_err(AppError::from)?; - Ok(Response::builder() - .status(StatusCode::CREATED) - .body(Body::empty()) - .unwrap()) + // Emit Content-Location (RFC 7231 §3.1.4.2) only when the URL + // canonicalization actually changed something — keeps the common + // ASCII / already-NFC path clean. Well-behaved clients (NC desktop, + // rclone) update their local index; naive clients ignore the + // header safely (status stays 201). + let mut response = Response::builder().status(StatusCode::CREATED); + if normalized_target != *target_name { + let mut canonical_subpath = String::with_capacity(subpath.len()); + for seg in parent_segments { + canonical_subpath.push_str(seg); + canonical_subpath.push('/'); + } + canonical_subpath.push_str(&normalized_target); + response = response.header( + "Content-Location", + nc_collection_href(&user.username, &canonical_subpath), + ); + } + Ok(response.body(Body::empty()).unwrap()) } // ──────────────────── DELETE ──────────────────── diff --git a/src/interfaces/web/embedded.rs b/src/interfaces/web/embedded.rs new file mode 100644 index 00000000..398c7687 --- /dev/null +++ b/src/interfaces/web/embedded.rs @@ -0,0 +1,164 @@ +//! Compile-time-embedded static assets — the `bundled-assets` feature. +//! +//! When the feature is on, the SvelteKit build output at `static-dist/` +//! (repo root, per `frontend/svelte.config.js`'s `adapter-static`) is +//! baked into the binary via `rust-embed` at compile time. The two axum +//! handlers below (`serve_root` for the SPA + fallback, `serve_immutable` +//! for the `_app/immutable` cache-forever subtree) parallel the two +//! `ServeDir` instances the filesystem path uses in `super::mod`. +//! +//! **Precedence** — this module is only invoked when +//! `resolve_static_source(config)` returns `StaticSource::Embedded`. If +//! `OXICLOUD_STATIC_PATH` (or the default `./static/static-dist/`) +//! points at a real directory, the filesystem `ServeDir` path is used +//! instead — ops can still override embedded bytes for locale patches +//! or theming without a full rebuild. +//! +//! **Compression** — `rust-embed`'s `compression` feature stores each +//! embedded file deflate-compressed. Lazy decompression on first access +//! keeps the binary small (~4-5 MB for the current corpus) and the +//! runtime cost negligible: after warmup every file is cached. Response +//! compression is handled by `CompressionLayer` in the parent module — +//! same wire behaviour as the filesystem path for `Content-Encoding: +//! br|gzip|identity` clients. +//! +//! **Vite's precompressed siblings** (`.br` / `.gz`) are excluded from +//! the embed via the `#[exclude]` attributes below — they'd be dead +//! weight because the response compression on the wire already handles +//! this negotiation. + +use axum::body::Body; +use axum::extract::{Path, Request}; +use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE}; +use axum::http::{HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; + +/// SvelteKit SPA build output, baked into the binary at compile time. +/// +/// The `#[folder]` path is relative to `Cargo.toml` (repo root), which +/// matches SvelteKit's `adapter-static` output in +/// `frontend/svelte.config.js` (`pages: '../static-dist'`). +#[derive(rust_embed::RustEmbed)] +#[folder = "static-dist/"] +// Default (no `include` attr) = include everything recursively. An +// explicit `include = "*"` was WRONG — the `*` glob is single-segment +// only, so `locales/*.json`, `_app/immutable/**`, and every other +// subdirectory got excluded and the boot-time `extract_embedded_locales` +// found 0 files (2026-08-28 bug fix). +// +// Vite's precompressed siblings — we serve raw and let the axum +// `CompressionLayer` on the wire negotiate br/gzip. Doubling storage +// would balloon the embed by ~50%. +#[exclude = "**/*.br"] +#[exclude = "**/*.gz"] +pub struct EmbeddedAssets; + +/// Serve any embedded asset by request path, falling back to the SPA +/// shell (`index.html`) for unmatched client routes. +/// +/// Mirror of the `spa` `ServeDir` in `super::create_web_routes` — same +/// fallback semantics so deep links like `/files/` boot the SvelteKit +/// router. `Cache-Control` for the shell itself is left to the outer +/// layer in the parent module (`no-cache` so a deploy can't leave a +/// stale app pinned in browsers); assets carrying no cache header here +/// pick up the parent's default the same way filesystem-served assets do. +/// +/// Wired as axum's `fallback` in `super::web_routes_embedded`, which +/// means there is NO route pattern to capture from — a `Path` extractor +/// would fail at runtime with "Wrong number of path arguments for +/// `Path`. Expected 1 but got 0." (real bug hit 2026-08-28). Pull the +/// URI path off the `Request` directly instead. +pub async fn serve_root(req: Request) -> Response { + let path = req.uri().path().trim_start_matches('/'); + if path.is_empty() { + return spa_shell_response(); + } + match EmbeddedAssets::get(path) { + Some(file) => asset_response(path, file.data), + None => spa_shell_response(), + } +} + +/// Root path (no trailing capture) — always the SPA shell. +/// +/// axum routes `/` separately from `/*path`, so this handles the +/// bare-slash case that `serve_root` never sees. +pub async fn serve_root_index() -> Response { + spa_shell_response() +} + +/// Serve an asset under the `/_app/immutable/*` prefix. The nested route +/// registration in `super::create_web_routes` already strips the +/// `/_app/immutable/` prefix from the captured path, so we look the +/// stripped path up with the prefix re-attached before hitting the embed. +/// +/// Cache-Control (`public, max-age=31536000, immutable`) is applied by +/// the outer `SetResponseHeaderLayer::overriding` in the parent module, +/// same as the filesystem path — this handler just returns bytes + MIME. +pub async fn serve_immutable(Path(path): Path) -> Response { + let full = format!("_app/immutable/{}", path.trim_start_matches('/')); + match EmbeddedAssets::get(&full) { + Some(file) => asset_response(&path, file.data), + None => (StatusCode::NOT_FOUND, "Not Found").into_response(), + } +} + +fn asset_response(path: &str, bytes: std::borrow::Cow<'static, [u8]>) -> Response { + let mime = mime_guess::from_path(path).first_or_octet_stream(); + let mut resp = Response::new(Body::from(bytes.into_owned())); + resp.headers_mut().insert( + CONTENT_TYPE, + HeaderValue::from_str(mime.as_ref()) + .unwrap_or(HeaderValue::from_static("application/octet-stream")), + ); + resp +} + +fn spa_shell_response() -> Response { + match EmbeddedAssets::get("index.html") { + Some(shell) => { + let mut resp = Response::new(Body::from(shell.data.into_owned())); + resp.headers_mut().insert( + CONTENT_TYPE, + HeaderValue::from_static("text/html; charset=utf-8"), + ); + // Belt: the parent module also stamps this on unset, + // but stamp it here too so the shell never accidentally + // ends up cacheable in front of a deploy. + resp.headers_mut() + .insert(CACHE_CONTROL, HeaderValue::from_static("no-cache")); + resp + } + None => { + // This means `static-dist/` was empty at build time — the + // build.rs guard should have prevented us from ever getting + // here. Surface as 500 rather than pretending the SPA works. + ( + StatusCode::INTERNAL_SERVER_ERROR, + "SPA shell missing from embedded assets; rebuild with an up-to-date static-dist/", + ) + .into_response() + } + } +} + +/// Iterate over the embedded `.html` files at the root of `static-dist/` +/// so the CSP inline-script scanner can hash them without a filesystem +/// read. Mirrors what `std::fs::read_dir(static_path)` yields on the +/// filesystem path, filtered to `.html` at the top level. +pub fn embedded_html_shells() -> Vec<(String, std::borrow::Cow<'static, [u8]>)> { + EmbeddedAssets::iter() + .filter(|p| { + // Root-level `.html` only — SvelteKit emits `index.html` at + // the root and everything else under `_app/`. Nested `.html` + // (e.g. sourcemap tooling artefacts) doesn't inline-script, + // so skip. + let s: &str = p.as_ref(); + s.ends_with(".html") && !s.contains('/') + }) + .filter_map(|p| { + let name = p.to_string(); + EmbeddedAssets::get(&name).map(|f| (name, f.data)) + }) + .collect() +} diff --git a/src/interfaces/web/mod.rs b/src/interfaces/web/mod.rs index e37a47b5..0f99647f 100644 --- a/src/interfaces/web/mod.rs +++ b/src/interfaces/web/mod.rs @@ -15,23 +15,105 @@ use tower_http::compression::CompressionLayer; use tower_http::services::{ServeDir, ServeFile}; use tower_http::set_header::SetResponseHeaderLayer; -/// Resolve the directory the SPA is actually served from. +#[cfg(feature = "bundled-assets")] +pub mod embedded; + +/// Where the SPA + immutable assets are served from — filesystem +/// (default and also the fallback on bundled builds when +/// `OXICLOUD_STATIC_PATH` points at real files) or the compile-time +/// embedded corpus (bundled-assets feature only). /// -/// Prefers the Vite build output (`static-dist/`) sitting next to the configured -/// static path, falling back to the configured path itself — the container ships -/// the built SPA straight to `OXICLOUD_STATIC_PATH` (default `./static`), so there -/// the fallback is what serves. Shared with the CSP layer in `main.rs` so the -/// inline-script hashes are computed from exactly the bytes that get served. -pub fn resolve_static_path(config: &AppConfig) -> PathBuf { +/// Returned by [`resolve_static_source`]; matched at each of the four +/// consumer sites (SPA `ServeDir`, `_app/immutable` `ServeDir`, +/// CSP inline-script scan, and the locale-loader picker in `main.rs`). +#[derive(Debug, Clone)] +pub enum StaticSource { + Filesystem(PathBuf), + /// Serve from the `EmbeddedAssets` corpus in the [`embedded`] module. + /// Only reachable under `--features bundled-assets` — the variant is + /// cfg-gated so match arms in non-bundled builds stay exhaustive on + /// a single variant, giving zero runtime cost. + #[cfg(feature = "bundled-assets")] + Embedded, +} + +/// Resolve where static assets come from. +/// +/// Order of precedence (highest first): +/// 1. `/../static-dist/` when it exists — matches +/// the SvelteKit adapter-static output at the repo root. +/// 2. `OXICLOUD_STATIC_PATH` itself when it exists — Docker image path +/// (assets copied straight to `/app/static/`). +/// 3. Bundled-assets fallback (only when the feature is on) — the +/// compile-time embedded corpus. +/// 4. Non-bundled fallback — return the configured path anyway, letting +/// downstream `ServeDir` fail predictably at request time. +/// +/// Rule (2) exists so ops running a bundled binary can still point +/// `OXICLOUD_STATIC_PATH` at a live directory (locale patch, theme +/// override) and see it win over the embedded copy without a rebuild. +pub fn resolve_static_source(config: &AppConfig) -> StaticSource { let dist = config .static_path .parent() .unwrap_or(Path::new(".")) .join("static-dist"); if dist.exists() { - return dist; + tracing::info!( + source = %dist.display(), + "static-assets: serving from filesystem (Vite build output at /../static-dist/)" + ); + return StaticSource::Filesystem(dist); + } + if config.static_path.exists() { + tracing::info!( + source = %config.static_path.display(), + "static-assets: serving from filesystem (OXICLOUD_STATIC_PATH)" + ); + return StaticSource::Filesystem(config.static_path.clone()); + } + #[cfg(feature = "bundled-assets")] + { + tracing::info!( + configured_static_path = %config.static_path.display(), + "static-assets: no filesystem source found, serving embedded corpus \ + (bundled-assets feature). Set OXICLOUD_STATIC_PATH to override with a \ + live directory." + ); + StaticSource::Embedded + } + #[cfg(not(feature = "bundled-assets"))] + { + // Non-bundled build with no on-disk source. Return the configured + // path anyway — downstream `ServeDir` will fail predictably at + // request time. A separate boot-time warning wouldn't help; the + // real fix is to build the SPA or set OXICLOUD_STATIC_PATH. + StaticSource::Filesystem(config.static_path.clone()) + } +} + +/// Backwards-compat helper: resolve to a `PathBuf` directly. +/// +/// Preserved for callers that predate the `StaticSource` enum. Only +/// callable in configurations where a filesystem path exists — a +/// bundled build whose `resolve_static_source` returned `Embedded` +/// would panic here, so new code should always match on +/// `resolve_static_source(...)` instead. +pub fn resolve_static_path(config: &AppConfig) -> PathBuf { + match resolve_static_source(config) { + StaticSource::Filesystem(p) => p, + #[cfg(feature = "bundled-assets")] + StaticSource::Embedded => { + // Every migrated caller matches on StaticSource directly; + // this branch means someone called the legacy helper from + // a bundled build. Fix the caller, not the shim. + panic!( + "resolve_static_path() called on a bundled build with no filesystem \ + assets — migrate the caller to resolve_static_source() and match on \ + StaticSource::Embedded" + ) + } } - config.static_path.clone() } /// Serves the SvelteKit single-page app. @@ -44,43 +126,33 @@ pub fn resolve_static_path(config: &AppConfig) -> PathBuf { /// Caching: content-hashed assets under `/_app/immutable` are cached forever; /// everything else — crucially the `index.html` shell — is `no-cache` so a deploy /// can't leave a stale app pinned in browsers. -pub fn create_web_routes(app_state: Arc) -> Router> { - let config = AppConfig::from_env(); - let static_path = resolve_static_path(&config); +pub fn create_web_routes(app_state: Arc, source: StaticSource) -> Router> { + // `source` is resolved ONCE at boot in `main.rs::run()` and passed + // in — see the sequence there. Previously this fn called + // `AppConfig::from_env()` + `resolve_static_source(&config)` itself + // (duplicating the env parse + storage-summary log). Threading the + // resolved value through as an arg keeps this fn pure and eliminates + // both duplicates in the boot log (bug fixed 2026-08-28). - // SPA fallback: serve the file if it exists, else the app shell. - // - // `precompressed_*`: if the frontend build emitted a sibling `.br`/`.gz` - // (frontend/scripts/precompress.mjs runs at build time), serve those - // bytes directly with the right Content-Encoding instead of re-running - // Brotli over the same immutable bundle on EVERY request — the - // `CompressionLayer` below then skips the already-encoded response and - // remains only the fallback for assets without a precompressed sibling - // (benches/STATIC-PRECOMPRESSED.md). - let spa = ServeDir::new(&static_path) - .precompressed_br() - .precompressed_gzip() - .fallback(ServeFile::new(static_path.join("index.html"))); + // Build the router — two shapes depending on `StaticSource`, but both + // wear the SAME outer layers below (compression fallback, no-cache + // default for the shell, OIDC login short-circuit). Keeping the + // layers common means the filesystem and embedded paths behave + // identically at the wire boundary. + let inner = match source { + StaticSource::Filesystem(static_path) => web_routes_filesystem(&static_path), + #[cfg(feature = "bundled-assets")] + StaticSource::Embedded => web_routes_embedded(), + }; - // Hashed, immutable assets (SvelteKit emits these under /_app/immutable). - let app_immutable = ServeDir::new(static_path.join("_app").join("immutable")) - .precompressed_br() - .precompressed_gzip(); - - Router::new() - .nest_service( - "/_app/immutable", - get_service(app_immutable).layer(SetResponseHeaderLayer::overriding( - CACHE_CONTROL, - HeaderValue::from_static("public, max-age=31536000, immutable"), - )), - ) - .fallback_service(spa) - // Fallback compression for assets without a precompressed sibling. - // Quality 4, NOT the default: the default maps to Brotli q11 — - // ~1.3 s of CPU per 700 KiB bundle per request (measured in - // benches/STATIC-PRECOMPRESSED.md; the .br siblings above carry the - // real q11 bytes, paid once at build time). + inner + // Fallback compression for assets without a precompressed sibling + // (filesystem) or for embedded assets that were compressed at + // compile time and decompressed on read (bundled). Quality 4, + // NOT the default: the default maps to Brotli q11 — ~1.3 s of + // CPU per 700 KiB bundle per request (benches/STATIC-PRECOMPRESSED.md; + // the .br siblings on the filesystem path carry the real q11 + // bytes, paid once at build time). .layer( CompressionLayer::new() .quality(tower_http::CompressionLevel::Precise(4)) @@ -105,6 +177,58 @@ pub fn create_web_routes(app_state: Arc) -> Router> { )) } +/// Filesystem-served SPA — the historical shape. Two `ServeDir` instances +/// with tower-http's `precompressed_br().precompressed_gzip()` picking up +/// Vite's precompressed siblings when present. +fn web_routes_filesystem(static_path: &Path) -> Router> { + // SPA fallback: serve the file if it exists, else the app shell. + // + // `precompressed_*`: if the frontend build emitted a sibling `.br`/`.gz` + // (frontend/scripts/precompress.mjs runs at build time), serve those + // bytes directly with the right Content-Encoding instead of re-running + // Brotli over the same immutable bundle on EVERY request — the + // `CompressionLayer` in `create_web_routes` then skips the already-encoded + // response and remains only the fallback for assets without a + // precompressed sibling (benches/STATIC-PRECOMPRESSED.md). + let spa = ServeDir::new(static_path) + .precompressed_br() + .precompressed_gzip() + .fallback(ServeFile::new(static_path.join("index.html"))); + + // Hashed, immutable assets (SvelteKit emits these under /_app/immutable). + let app_immutable = ServeDir::new(static_path.join("_app").join("immutable")) + .precompressed_br() + .precompressed_gzip(); + + Router::new() + .nest_service( + "/_app/immutable", + get_service(app_immutable).layer(SetResponseHeaderLayer::overriding( + CACHE_CONTROL, + HeaderValue::from_static("public, max-age=31536000, immutable"), + )), + ) + .fallback_service(spa) +} + +/// Embedded-assets SPA — mirror of `web_routes_filesystem` using the +/// [`embedded`] module's handlers instead of `ServeDir`. Same URL shape, +/// same cache-header layers, same SPA-shell fallback semantics. +#[cfg(feature = "bundled-assets")] +fn web_routes_embedded() -> Router> { + use axum::routing::get; + Router::new() + .route( + "/_app/immutable/{*path}", + get(embedded::serve_immutable).layer(SetResponseHeaderLayer::overriding( + CACHE_CONTROL, + HeaderValue::from_static("public, max-age=31536000, immutable"), + )), + ) + .route("/", get(embedded::serve_root_index)) + .fallback(get(embedded::serve_root)) +} + /// Intercept `GET /login` and 302 to `/api/auth/oidc/authorize` when OIDC is /// the only working method (see `AuthApplicationService::auto_redirect_to_oidc`). /// @@ -161,15 +285,31 @@ async fn oidc_standalone_login_redirect( /// web worker from a blob URL; `'self'` covers same-origin workers like the /// delta-upload worker. pub fn content_security_policy(config: &AppConfig) -> String { - let static_path = resolve_static_path(config); - let hashes = inline_script_csp_hashes(&static_path); + let source = resolve_static_source(config); + let hashes = match &source { + StaticSource::Filesystem(p) => inline_script_csp_hashes(p), + #[cfg(feature = "bundled-assets")] + StaticSource::Embedded => inline_script_csp_hashes_embedded(), + }; if hashes.is_empty() { - tracing::warn!( - static_path = %static_path.display(), - "CSP: no inline