Merge pull request #651 from EdouardVanbelle/feat/recoverable-jobs
feat(recoverable jobs): add engine + consistency jobs
This commit is contained in:
Dionisio Pozo
@@ -172,6 +172,7 @@ export default defineConfig({
|
||||
{ text: "User lifecycle", link: "/architecture/user-lifecycle" },
|
||||
{ text: "Authentication model", link: "/architecture/auth-model" },
|
||||
{ text: "Magic-link auth", link: "/architecture/magic-link-auth" },
|
||||
{ text: "Background jobs", link: "/architecture/jobs" },
|
||||
],
|
||||
},
|
||||
{ text: "FAQ", link: "/faq" },
|
||||
|
||||
@@ -74,3 +74,4 @@ src/
|
||||
- [Caching Architecture →](/architecture/caching)
|
||||
- [Resource Listing API →](/architecture/resource-listing)
|
||||
- [Storage Quotas →](/architecture/storage-quotas)
|
||||
- [Background Jobs →](/architecture/jobs)
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
# Background Jobs
|
||||
|
||||
OxiCloud runs periodic maintenance work through a central **JobRegistry**
|
||||
scheduler. This document is for implementors adding a new tenant or
|
||||
debugging an existing one.
|
||||
|
||||
The design rationale (why a registry vs. per-service `tokio::spawn`
|
||||
loops, the two-engine split, plugin future) lives in the plan doc:
|
||||
[`docs/plan/job-registry.md`](../plan/job-registry.md). This page
|
||||
is the "how to plug in" reference.
|
||||
|
||||
---
|
||||
|
||||
## When does a background loop belong here?
|
||||
|
||||
Single decision question:
|
||||
|
||||
> **"Would an operator plausibly `POST /api/admin/jobs/{name}/trigger`
|
||||
> to make it run right now?"**
|
||||
|
||||
If yes, register it with JobRegistry. You get:
|
||||
|
||||
- Uniform admin surface (list, trigger, last-outcome).
|
||||
- Uniform `oxicloud::scheduler` log line per run with `elapsed_ms`.
|
||||
- Panic containment (a handler that panics doesn't kill the scheduler).
|
||||
- Exclusivity: only one in-flight run per job name (a tick that fires
|
||||
while the previous run is still going is skipped, with a warning).
|
||||
- Optional wall-clock timeout.
|
||||
- Optional acceleration parameter (`?force=true` → `JobRunArgs.force`).
|
||||
|
||||
If no — the loop is a queue drainer (`tree_etag_flush_job`), a
|
||||
continuous event reactor (`content_index_worker`), or a stats printer
|
||||
(`db_pool_monitor`) — keep it as its own dedicated `tokio::spawn`
|
||||
loop. Wedging it into JobRegistry adds framework overhead for no
|
||||
operator benefit.
|
||||
|
||||
---
|
||||
|
||||
## Current tenants
|
||||
|
||||
| Job name | Cadence | Force semantic | Service |
|
||||
|---|---|---|---|
|
||||
| `trash_cleanup` | 24 h (hardcoded in DI, no env var yet) | ignored | [`trash_cleanup_service.rs`](../../src/infrastructure/services/trash_cleanup_service.rs) |
|
||||
| `storage_reconcile`| `OXICLOUD_STORAGE_USAGE_RECONCILE_SECS` (default 600s, min 30s) | ignored | [`storage_usage_service.rs`](../../src/application/services/storage_usage_service.rs) |
|
||||
| `dedup_gc` | on-demand only (trash cleanup runs it inline as its tail step) | `force=true` → `garbage_collect_force()` (skip orphan grace) | [`dedup_service.rs`](../../src/infrastructure/services/dedup_service.rs) |
|
||||
| `grant_cleanup` | `OXICLOUD_GRANT_CLEANUP_INTERVAL_HOURS` (default 24h) — feature-gated by `OXICLOUD_GRANT_CLEANUP_ENABLED` | `force=true` → `purge(Some(0))` (grace_days=0) | [`grant_cleanup_service.rs`](../../src/infrastructure/services/grant_cleanup_service.rs) |
|
||||
|
||||
---
|
||||
|
||||
## Adding a new job — recipe
|
||||
|
||||
### 1. Implement `JobHandler` on your service
|
||||
|
||||
```rust
|
||||
use std::sync::Arc;
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs};
|
||||
|
||||
pub const MY_JOB_NAME: &str = "my_job"; // stable snake_case
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for MyService {
|
||||
fn name(&self) -> &str {
|
||||
MY_JOB_NAME
|
||||
}
|
||||
|
||||
async fn run(&self, args: &JobRunArgs) -> JobOutcome {
|
||||
// …do work…
|
||||
match self.do_the_work().await {
|
||||
Ok(count) => JobOutcome::ok_with(
|
||||
count,
|
||||
serde_json::json!({
|
||||
// Anything useful for `GET /api/admin/jobs` or logs.
|
||||
"bytes_processed": total_bytes,
|
||||
"forced": args.force,
|
||||
}),
|
||||
),
|
||||
Err(e) => JobOutcome::err(format!("my_job failed: {e}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Rules:
|
||||
- **`name()` must be stable** — it appears in log lines, admin URLs,
|
||||
and `admin.background_runs.job_name` (once Part 2 lands). Renaming
|
||||
is a breaking change to operator scripts and log dashboards.
|
||||
- **`args.force` semantics are per-job.** If your job has no
|
||||
acceleration mode (e.g. reconciliation, which is always idempotent),
|
||||
ignore it. If it does (e.g. skip a grace window), document the
|
||||
behaviour on the handler docstring.
|
||||
- **Return `JobOutcome::ok_with(count, extra)`.** `count` is the
|
||||
primary scalar operators read (rows swept, blobs reclaimed).
|
||||
`extra` is a free-form JSON blob surfaced in the log line and the
|
||||
admin listing.
|
||||
- **Return `JobOutcome::err(msg)` on failure.** The supervisor logs
|
||||
`outcome=err, cause=handler` and continues to the next tick. Don't
|
||||
catch panics inside `run()` — the supervisor does it, and hiding
|
||||
one loses the `cause=panicked` diagnostic.
|
||||
|
||||
### 2. Add a `register()` method to your service
|
||||
|
||||
Every service uses the chainable self-registration shape:
|
||||
|
||||
```rust
|
||||
impl MyService {
|
||||
/// Register self with the periodic-job scheduler and return the
|
||||
/// same `Arc<Self>` for DI-style chaining.
|
||||
pub async fn register(self: Arc<Self>, registry: &JobRegistry) -> Arc<Self> {
|
||||
let interval = /* Some(Duration::from_secs(...)) or None for on-demand */;
|
||||
registry.register(self.clone(), interval, None /* no timeout */).await;
|
||||
self
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- **Interval `Some(dur)`** → **scheduled**. The supervisor fires the
|
||||
handler every `dur`.
|
||||
- **Interval `None`** → **on-demand only**. Never fires periodically;
|
||||
only reachable via `POST /api/admin/jobs/{name}/trigger` or a
|
||||
programmatic call to `registry.trigger(name, args)`. Use for jobs
|
||||
whose periodic work happens elsewhere (dedup GC piggybacks on
|
||||
trash cleanup) but which still benefit from a uniform admin trigger.
|
||||
- **Timeout `Some(dur)`** → the supervisor wraps the handler in
|
||||
`tokio::time::timeout`. Timeout trip is logged as
|
||||
`outcome=err, cause=timeout`. Use sparingly — most native jobs don't
|
||||
need it. Note: aborting a task mid-run is best-effort; a handler
|
||||
that ignores await points may still finish in the background.
|
||||
|
||||
### 3. Wire in `common/di.rs`
|
||||
|
||||
One statement per service:
|
||||
|
||||
```rust
|
||||
let my_service = Arc::new(MyService::new(deps...))
|
||||
.register(&core.job_registry)
|
||||
.await;
|
||||
```
|
||||
|
||||
`register()` panics on error (duplicate job name = DI wiring bug;
|
||||
boot must fail loud) and emits a `job.registered` log line on
|
||||
success. No `if let Err(e)` scaffolding needed at the call site.
|
||||
|
||||
`core.job_registry` is populated inside `create_core_services` and
|
||||
lives on `CoreServices`. `SchedulerEngine` spawns the supervisor
|
||||
task at the end of `build_app_state` after every service has
|
||||
registered.
|
||||
|
||||
### 4. That's it
|
||||
|
||||
- `GET /api/admin/jobs` immediately lists the new job.
|
||||
- `POST /api/admin/jobs/my_job/trigger` runs one dispatch off-schedule.
|
||||
- `POST /api/admin/jobs/my_job/trigger?force=true` runs one dispatch
|
||||
with `JobRunArgs { force: true }`.
|
||||
- If the job is scheduled, the supervisor fires it at the configured
|
||||
cadence.
|
||||
|
||||
No handler registration in the router, no admin trigger endpoint to
|
||||
add — the framework already covers those uniformly.
|
||||
|
||||
---
|
||||
|
||||
## Admin surface
|
||||
|
||||
Production admin endpoints, always on, audit-logged. Admin-only via
|
||||
the standard `/api/admin/*` middleware (no dedicated feature flag).
|
||||
|
||||
```
|
||||
GET /api/admin/jobs
|
||||
→ [{ name, interval_ms?, next_run_at?, last_run_at?, last_outcome?, running }]
|
||||
|
||||
POST /api/admin/jobs/{name}/trigger[?force=<bool>]
|
||||
→ 200 { ok: true, outcome: { outcome: "ok" | "err", count, extra?, message? } }
|
||||
→ 404 { error: "job not registered", name }
|
||||
```
|
||||
|
||||
- `interval_ms` / `next_run_at` are absent for on-demand jobs
|
||||
(`skip_serializing_if=None`).
|
||||
- `last_run_at` / `last_outcome` are absent until the first run
|
||||
completes.
|
||||
- `running` is `true` iff the in-flight permit is currently held.
|
||||
- Every trigger call emits a `target: "audit"` log line
|
||||
(`event = "job.trigger"`) before dispatch.
|
||||
|
||||
---
|
||||
|
||||
## Log lines
|
||||
|
||||
Uniform target: `oxicloud::scheduler`.
|
||||
|
||||
| Event | Fields | When |
|
||||
|---|---|---|
|
||||
| `scheduler.started` | — | Supervisor loop starts |
|
||||
| `scheduler.ready` | `registered = N` | All services have registered |
|
||||
| `job.registered` | `job`, `cadence` ("every 24 h" / "on-demand") | Each `register()` call |
|
||||
| `job.run` | `job`, `outcome` (ok\|err), `cause?` (handler\|timeout\|panicked), `count`, `elapsed_ms`, `extra?`, `error?` | Every dispatch |
|
||||
| `job.tick_skipped` | `job`, `interval_ms`, `running_for_ms` | Tick fires while the previous run is still going |
|
||||
| `job.trigger` | `job`, `force` (audit channel) | Admin `POST /trigger` |
|
||||
|
||||
`elapsed_ms` is the raw scalar in the structured field; the human
|
||||
message renders it as `12ms` / `1.4s` / `4m30s` so `tail -f` operators
|
||||
see the duration inline.
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
The scheduler's own tests live in `src/infrastructure/scheduler/`
|
||||
(`registry.rs::tests`, `engine.rs::tests`) and use dummy handlers.
|
||||
No integration test infrastructure is needed to add a new service —
|
||||
your service's normal unit tests cover the `run()` logic, and the
|
||||
Hurl suite `tests/api/admin_jobs.hurl` covers the admin surface
|
||||
generically.
|
||||
|
||||
If your service has a test that needs to construct the type WITHOUT
|
||||
registering with a scheduler (e.g. isolated unit tests), skip the
|
||||
`.register(®).await` chain and use the bare `Arc::new(Service::new(...))`.
|
||||
|
||||
---
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **Cross-job dependencies.** No `depends_on` — each job runs
|
||||
independently. If you find yourself needing "job B runs after job A
|
||||
completes", route the completion signal through a lifecycle hook
|
||||
(`FileLifecycleHook`, `BlobLifecycleHook`), not through the scheduler.
|
||||
- **Distributed scheduling.** Single-process only. If OxiCloud ever
|
||||
runs multi-node, the pattern is `SELECT … FOR UPDATE SKIP LOCKED`
|
||||
on a lease table — not this design.
|
||||
- **Cron expressions.** Fixed intervals only. Real cron
|
||||
(day-of-week/month, arbitrary times) can layer on top later via a
|
||||
`next_run: Box<dyn NextRun>` trait; nothing needs it today.
|
||||
- **Backfill on startup.** If the process was down when a scheduled
|
||||
tick was due, the missed tick is NOT caught up — the next tick fires
|
||||
at its normal interval.
|
||||
|
||||
---
|
||||
|
||||
## Related
|
||||
|
||||
- Plan doc: [`docs/plan/job-registry.md`](../plan/job-registry.md)
|
||||
- Long-running / resumable jobs (Part 2, not yet built):
|
||||
[`docs/plan/job-registry.md#part-2--recoverable-run-engine`](../plan/job-registry.md#part-2--recoverable-run-engine)
|
||||
- Consistency checks (a future Part 2 consumer):
|
||||
[`docs/plan/consistency-check.md`](../plan/consistency-check.md)
|
||||
@@ -1,5 +1,25 @@
|
||||
# Plan — Resumable consistency checks + `StatefulAdapter` contract
|
||||
|
||||
> ⚠️ **PARTIALLY SUPERSEDED (Ed 2026-07-28).** The current shipping
|
||||
> design organises consistency checks **by the subject they iterate**
|
||||
> (drives / folders / files / storage), NOT by concern (blob / thumbnail
|
||||
> / used_bytes). Each `*_consistency` job is a direct
|
||||
> `RecoverableJobHandler` impl on the Part 2 engine — no
|
||||
> `ConsistencyCheck` trait, no `StatefulAdapter` supertrait, no per-
|
||||
> subsystem check registry. Cursor = row PK of the iterated subject.
|
||||
>
|
||||
> **See instead:**
|
||||
> - Memory: `project_consistency_jobs_landscape` — the current taxonomy.
|
||||
> - `docs/plan/job-registry.md` Part 2 §Native tenants — updated table.
|
||||
> - `docs/architecture/jobs.md` — implementor guide.
|
||||
>
|
||||
> Sections below discuss `BlobConsistencyCheck`, `ThumbnailConsistencyCheck`,
|
||||
> `UsedBytesConsistencyCheck` etc. as separate impls of a
|
||||
> `ConsistencyCheck` trait. That IS retired. Read those sections for
|
||||
> the invariants (grace-window trap, cursor discipline, findings
|
||||
> idempotency) — they still apply. Ignore the trait shapes /
|
||||
> registration wiring — the Part 2 engine covers those uniformly.
|
||||
|
||||
## Context
|
||||
|
||||
OxiCloud persists state in several independent subsystems: content-addressable
|
||||
@@ -38,14 +58,14 @@ This plan lands:
|
||||
compiles without declaring its consistency contract.
|
||||
3. An **educational surface** in trait doc-comments — decision axes
|
||||
(severity, direction, grace, cursor) and canonical-example pointers.
|
||||
4. **Consistency-specific persistence** — `admin.consistency_findings`,
|
||||
4. **Consistency-specific persistence** — `jobs.run_findings`,
|
||||
idempotent-on-`(run_id, kind, resource_id)`.
|
||||
5. A **first check** — `BlobConsistencyCheck` (both directions,
|
||||
blob-keyed cursor, severity split).
|
||||
|
||||
**Layer boundary — the runtime is not in this plan.** The resumable
|
||||
execution engine (cursor persistence, exclusivity, cancel protocol,
|
||||
crash recovery, `admin.background_runs` schema, `JobStore`,
|
||||
crash recovery, `jobs.recoverable_runs` schema, `JobStore`,
|
||||
`RunOutcome`, `run_or_resume`) lives in `docs/plan/job-registry.md`
|
||||
Part 2. This plan describes what `ConsistencyCheck` implementors
|
||||
write and how the check-specific bits (findings, severity,
|
||||
@@ -53,7 +73,7 @@ write and how the check-specific bits (findings, severity,
|
||||
|
||||
**Order:** ships **after** the job-registry Part 2 engine lands.
|
||||
Consistency closes an operator-visibility gap today, but it depends
|
||||
on Part 2's `RecoverableJob` + `JobStore` + `admin.background_runs`
|
||||
on Part 2's `RecoverableJobHandler` + `JobStore` + `jobs.recoverable_runs`
|
||||
primitives — those come first. Once both are in, consistency runs
|
||||
are admin-triggered v1, becoming periodic-triggered when a
|
||||
`JobRegistry` (Part 1) tenant wraps `run_or_resume` for each
|
||||
@@ -157,11 +177,11 @@ Race matrix — missing direction:
|
||||
Nothing before "byte-exact whole-table snapshot verification" needs a
|
||||
quiescent server. Reserve `concurrent_safe() = false` for that one.
|
||||
|
||||
### Resumability — runs live in Part 2's `background_runs`
|
||||
### Resumability — runs live in Part 2's `recoverable_runs`
|
||||
|
||||
Consistency runs are ordinary `RecoverableJob`s. The runtime plumbing
|
||||
Consistency runs are ordinary `RecoverableJobHandler`s. The runtime plumbing
|
||||
— cursor persistence, exclusivity, cancel protocol, crash recovery,
|
||||
`admin.background_runs` schema, `JobStore` trait, `RunOutcome`,
|
||||
`jobs.recoverable_runs` schema, `JobStore` trait, `RunOutcome`,
|
||||
`run_or_resume` helper — lives in `docs/plan/job-registry.md` Part 2.
|
||||
This plan does not redefine any of it.
|
||||
|
||||
@@ -172,10 +192,10 @@ This plan does not redefine any of it.
|
||||
`SELECT DISTINCT ON (job_name)` return the last run of every check
|
||||
alongside every other background job.
|
||||
- Consistency's per-check knobs — `grace_window_secs`, `batch_size`,
|
||||
`concurrent_safe` — live inside `background_runs.params` JSONB at
|
||||
`concurrent_safe` — live inside `recoverable_runs.params` JSONB at
|
||||
run-start time. The check reads them back via
|
||||
`serde_json::from_value(store.params()?)`.
|
||||
- `background_runs.stats` accumulates `{"scanned_count": …,
|
||||
- `recoverable_runs.stats` accumulates `{"scanned_count": …,
|
||||
"findings_this_run": …}`; readers call
|
||||
`(stats->>'scanned_count')::bigint`.
|
||||
|
||||
@@ -183,9 +203,9 @@ The findings themselves are Layer C (this plan) — they don't
|
||||
generalise to storage-migration or reextract:
|
||||
|
||||
```sql
|
||||
CREATE TABLE admin.consistency_findings (
|
||||
CREATE TABLE jobs.run_findings (
|
||||
id UUID PRIMARY KEY,
|
||||
run_id UUID NOT NULL REFERENCES admin.background_runs(id) ON DELETE CASCADE,
|
||||
run_id UUID NOT NULL REFERENCES jobs.recoverable_runs(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL, -- OrphanBlob / MissingBlob / ...
|
||||
severity TEXT NOT NULL, -- DataLoss / Reclaimable / ...
|
||||
resource_id TEXT NOT NULL,
|
||||
@@ -193,15 +213,15 @@ CREATE TABLE admin.consistency_findings (
|
||||
found_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (run_id, kind, resource_id) -- idempotent re-scan on resume
|
||||
);
|
||||
CREATE INDEX ON admin.consistency_findings (run_id, severity);
|
||||
CREATE INDEX ON jobs.run_findings (run_id, severity);
|
||||
```
|
||||
|
||||
FK on `background_runs.id` links a finding back to the run that
|
||||
FK on `recoverable_runs.id` links a finding back to the run that
|
||||
produced it; `ON DELETE CASCADE` clears findings when their run row
|
||||
is pruned by a future retention job.
|
||||
|
||||
`admin.*` is a NEW schema, created by Part 2's migration — keep it
|
||||
distinct from `auth.*` / `storage.*` so operational tables don't
|
||||
`jobs.*` is a NEW schema, created by Part 2's migration — keep it
|
||||
distinct from `auth.*` / `storage.*` / `admin.*` so operational tables don't
|
||||
pollute domain schemas.
|
||||
|
||||
### Non-obvious traps
|
||||
@@ -223,7 +243,7 @@ learned the hard way in similar systems:
|
||||
transitioned (was `MissingBlob`, blob has since landed → drop the
|
||||
finding, not the whole run).
|
||||
4. **Cooperative cancellation ONLY.** Between batches, poll
|
||||
`background_runs.status`. A `tokio::spawn` abort mid-batch leaks —
|
||||
`recoverable_runs.status`. A `tokio::spawn` abort mid-batch leaks —
|
||||
cursor unpersisted, findings half-written. Cancel path writes
|
||||
`status='Paused'` + current cursor before returning.
|
||||
5. **Crash recovery on boot.** Any `status='Running'` at server start =
|
||||
@@ -484,7 +504,7 @@ impl ConsistencyRegistry {
|
||||
|
||||
## Admin surface
|
||||
|
||||
Consistency runs are ordinary `RecoverableJob`s (see
|
||||
Consistency runs are ordinary `RecoverableJobHandler`s (see
|
||||
`docs/plan/job-registry.md` Part 2), so most operator actions reach
|
||||
them through the shared scheduler surface:
|
||||
|
||||
@@ -510,7 +530,7 @@ GET /api/admin/jobs/consistency_{name}/runs/{id}
|
||||
```
|
||||
|
||||
Findings enrichment on `runs/{id}` is consistency-specific — read
|
||||
from `admin.consistency_findings` and joined into the response.
|
||||
from `jobs.run_findings` and joined into the response.
|
||||
Everything else is generic Part 2 behaviour.
|
||||
|
||||
Production surface — always on, audit-logged. No feature-flag gate.
|
||||
@@ -528,22 +548,22 @@ Production surface — always on, audit-logged. No feature-flag gate.
|
||||
`src/infrastructure/services/consistency/mod.rs`
|
||||
- `ConsistencyRegistry` (data structure only).
|
||||
- `PgCheckStore` — impl of `CheckStore` reading/writing
|
||||
`admin.background_runs` (filtered to `job_name LIKE 'consistency_%'`)
|
||||
+ `admin.consistency_findings`.
|
||||
`jobs.recoverable_runs` (filtered to `job_name LIKE 'consistency_%'`)
|
||||
+ `jobs.run_findings`.
|
||||
- `run_check(check, cursor, store)` — the runner that calls
|
||||
`run_resumable`, applies timeout, records outcome.
|
||||
|
||||
### 2. Schema migration
|
||||
|
||||
`migrations/YYYYMMDDHHMMSS_background_runs_admin_schema.sql` — creates
|
||||
the merged `admin.background_runs` table shared with the JobRegistry
|
||||
plan. Consistency checks own the `admin.consistency_findings` table
|
||||
alone and reference `background_runs.id` via FK.
|
||||
the merged `jobs.recoverable_runs` table shared with the JobRegistry
|
||||
plan. Consistency checks own the `jobs.run_findings` table
|
||||
alone and reference `recoverable_runs.id` via FK.
|
||||
|
||||
```sql
|
||||
CREATE SCHEMA IF NOT EXISTS admin;
|
||||
|
||||
CREATE TABLE admin.background_runs (
|
||||
CREATE TABLE jobs.recoverable_runs (
|
||||
id UUID PRIMARY KEY,
|
||||
job_name TEXT NOT NULL, -- 'consistency_blobs', 'storage_migration', 'reextract_audio', ...
|
||||
status TEXT NOT NULL, -- Running / Paused / Completed / Failed / CancelRequested
|
||||
@@ -556,13 +576,13 @@ CREATE TABLE admin.background_runs (
|
||||
error_message TEXT
|
||||
);
|
||||
CREATE UNIQUE INDEX one_active_run_per_job
|
||||
ON admin.background_runs (job_name)
|
||||
ON jobs.recoverable_runs (job_name)
|
||||
WHERE status IN ('Running', 'Paused');
|
||||
CREATE INDEX ON admin.background_runs (last_progress_at) WHERE status = 'Running';
|
||||
CREATE INDEX ON jobs.recoverable_runs (last_progress_at) WHERE status = 'Running';
|
||||
|
||||
CREATE TABLE admin.consistency_findings (
|
||||
CREATE TABLE jobs.run_findings (
|
||||
id UUID PRIMARY KEY,
|
||||
run_id UUID NOT NULL REFERENCES admin.background_runs(id) ON DELETE CASCADE,
|
||||
run_id UUID NOT NULL REFERENCES jobs.recoverable_runs(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL,
|
||||
severity TEXT NOT NULL,
|
||||
resource_id TEXT NOT NULL,
|
||||
@@ -570,7 +590,7 @@ CREATE TABLE admin.consistency_findings (
|
||||
found_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (run_id, kind, resource_id)
|
||||
);
|
||||
CREATE INDEX ON admin.consistency_findings (run_id, severity);
|
||||
CREATE INDEX ON jobs.run_findings (run_id, severity);
|
||||
```
|
||||
|
||||
### 3. Supertrait bounds on existing state-owning ports
|
||||
@@ -631,7 +651,7 @@ Ship this PR without the actual checks. Grep `TODO(consistency)` = punch list.
|
||||
`src/interfaces/api/handlers/admin_handler.rs`
|
||||
|
||||
- `start_consistency_check(name, force)` — insert an
|
||||
`admin.background_runs` row with `job_name = 'consistency_<name>'`
|
||||
`jobs.recoverable_runs` row with `job_name = 'consistency_<name>'`
|
||||
and `status = 'Running'`, spawn a tokio task calling `run_check`,
|
||||
return `run_id`. Concurrent triggers hit the partial unique index
|
||||
and short-circuit to returning the surviving row.
|
||||
@@ -651,7 +671,7 @@ In `AppServiceFactory` init, after DB pool is up:
|
||||
|
||||
```rust
|
||||
sqlx::query!(
|
||||
"UPDATE admin.background_runs
|
||||
"UPDATE jobs.recoverable_runs
|
||||
SET status = 'Paused',
|
||||
error_message = COALESCE(error_message, 'server restart mid-run')
|
||||
WHERE job_name LIKE 'consistency_%'
|
||||
@@ -660,7 +680,7 @@ sqlx::query!(
|
||||
```
|
||||
|
||||
Filtering on `job_name LIKE 'consistency_%'` scopes the sweep to
|
||||
consistency runs; other tenants of `background_runs` (storage
|
||||
consistency runs; other tenants of `recoverable_runs` (storage
|
||||
migration, reextract-*) run the same auto-Pause sweep from their own
|
||||
boot-time hook. The JobRegistry supervisor's boot check may
|
||||
generalise this into a single scheduler-wide sweep — until then, one
|
||||
|
||||
+185
-47
@@ -115,8 +115,8 @@ are already there. Keeping it as its own loop is right.
|
||||
of trash-cleanup + storage-usage + db_pool_monitor + dedup GC +
|
||||
grant-cleanup + tree-etag flush + content-index. High mechanical
|
||||
payoff, zero new schema, minimal review surface.
|
||||
2. **Part 2 lands next** — introduces `admin.background_runs` schema
|
||||
+ `RecoverableJob` trait + `JobStore` port + `run_or_resume`. On
|
||||
2. **Part 2 lands next** — introduces `jobs.recoverable_runs` schema
|
||||
+ `RecoverableJobHandler` trait + `JobStore` port + `run_or_resume`. On
|
||||
its own PR (schema change deserves independent review).
|
||||
3. **Consistency-check framework (`docs/plan/consistency-check.md`)**
|
||||
lands third, consuming Part 2 as its runtime.
|
||||
@@ -432,16 +432,16 @@ into the scheduler.
|
||||
|
||||
## Part 2 — Recoverable-Run Engine
|
||||
|
||||
### Contract — `RecoverableJob` trait
|
||||
### Contract — `RecoverableJobHandler` trait
|
||||
|
||||
Sibling to `JobHandler`, NOT a subtrait. A stateless job that only
|
||||
implements `JobHandler` never needs to know Part 2 exists.
|
||||
|
||||
```rust
|
||||
#[async_trait]
|
||||
pub trait RecoverableJob: Send + Sync {
|
||||
pub trait RecoverableJobHandler: Send + Sync {
|
||||
/// Stable snake_case identifier — matches the `job_name` column
|
||||
/// in `admin.background_runs`.
|
||||
/// in `jobs.recoverable_runs`.
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Long-running, cooperative scan. The store is the job's ONLY
|
||||
@@ -479,13 +479,13 @@ pub enum RunOutcome {
|
||||
### `JobStore` trait
|
||||
|
||||
The port the engine passes to a recoverable job. Backed by
|
||||
`admin.background_runs` in production; can be mocked for unit tests.
|
||||
`jobs.recoverable_runs` in production; can be mocked for unit tests.
|
||||
|
||||
```rust
|
||||
#[async_trait]
|
||||
pub trait JobStore: Send + Sync {
|
||||
/// The `run_id` this handler was invoked with. Uniquely identifies
|
||||
/// the row in `admin.background_runs`.
|
||||
/// the row in `jobs.recoverable_runs`.
|
||||
fn run_id(&self) -> Uuid;
|
||||
|
||||
/// Fixed at run start; used by consistency checks (and any other
|
||||
@@ -517,12 +517,12 @@ instance) are separate traits the impl composes on top of `JobStore`.
|
||||
`JobStore` itself carries no findings/severity concept — those are
|
||||
Layer C in the consistency-check plan, not the engine's concern.
|
||||
|
||||
### Schema — `admin.background_runs`
|
||||
### Schema — `jobs.recoverable_runs`
|
||||
|
||||
```sql
|
||||
CREATE SCHEMA IF NOT EXISTS admin;
|
||||
|
||||
CREATE TABLE admin.background_runs (
|
||||
CREATE TABLE jobs.recoverable_runs (
|
||||
id UUID PRIMARY KEY,
|
||||
job_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL, -- Running / Paused / CancelRequested / Completed / Failed
|
||||
@@ -536,10 +536,10 @@ CREATE TABLE admin.background_runs (
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX one_active_run_per_job
|
||||
ON admin.background_runs (job_name)
|
||||
ON jobs.recoverable_runs (job_name)
|
||||
WHERE status IN ('Running', 'Paused', 'CancelRequested');
|
||||
|
||||
CREATE INDEX ON admin.background_runs (last_progress_at)
|
||||
CREATE INDEX ON jobs.recoverable_runs (last_progress_at)
|
||||
WHERE status = 'Running';
|
||||
```
|
||||
|
||||
@@ -549,9 +549,9 @@ so it survives concurrent triggers, admin-vs-scheduler races, and
|
||||
transaction interleavings. The `CancelRequested` inclusion prevents
|
||||
a second trigger during cancel from spawning a parallel run.
|
||||
|
||||
`admin.*` is a NEW schema — kept distinct from `auth.*` / `storage.*`
|
||||
`jobs.*` is a NEW schema — kept distinct from `auth.*` / `storage.*` / `admin.*`
|
||||
so operational tables don't pollute domain schemas. Consistency
|
||||
checks own their own `admin.consistency_findings` in the same
|
||||
checks own their own `jobs.run_findings` in the same
|
||||
schema.
|
||||
|
||||
Cursor is `BYTEA`, not JSONB, because per-job cursors are fixed-shape
|
||||
@@ -579,7 +579,7 @@ One `UPDATE` per checkpoint. Cheap, no row-lock contention (this
|
||||
process owns the row):
|
||||
|
||||
```sql
|
||||
UPDATE admin.background_runs
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET cursor = $2,
|
||||
stats = jsonb_set(
|
||||
stats,
|
||||
@@ -618,7 +618,7 @@ that's a success. Log lines stay meaningful (`outcome=ok`,
|
||||
The engine module exposes:
|
||||
|
||||
```rust
|
||||
pub async fn run_or_resume<J: RecoverableJob + ?Sized>(
|
||||
pub async fn run_or_resume<J: RecoverableJobHandler + ?Sized>(
|
||||
job: Arc<J>,
|
||||
store_factory: &dyn JobStoreFactory,
|
||||
) -> JobOutcome
|
||||
@@ -671,7 +671,7 @@ At `AppServiceFactory` init, after DB pool is up:
|
||||
|
||||
```rust
|
||||
sqlx::query!(
|
||||
"UPDATE admin.background_runs
|
||||
"UPDATE jobs.recoverable_runs
|
||||
SET status = 'Paused',
|
||||
error_message = COALESCE(error_message, 'server restart mid-run')
|
||||
WHERE status IN ('Running', 'CancelRequested')"
|
||||
@@ -703,18 +703,28 @@ GET /api/admin/jobs/{name}/runs/{id}
|
||||
|
||||
### Native tenants (Part 2)
|
||||
|
||||
- **Blob storage backend migration.** `migration_job.rs` becomes a
|
||||
`RecoverableJob` impl. Cursor = last processed blob hash. Retires
|
||||
the `Arc<RwLock<MigrationState>>` in-memory struct.
|
||||
- **Reextract audio metadata.** Currently synchronous inside the
|
||||
admin HTTP request. Becomes a `RecoverableJob` iterating audio
|
||||
files by `file_id`.
|
||||
- **Reextract image/video capture dates.** Same as above.
|
||||
- **Consistency-check runs.** Every `ConsistencyCheck` impl gets
|
||||
wrapped by a `RecoverableJob` adapter; the wrapper writes to
|
||||
`admin.background_runs` via `JobStore`, and separately writes
|
||||
findings to `admin.consistency_findings` via a check-specific
|
||||
extension trait. See `docs/plan/consistency-check.md`.
|
||||
Consistency checks are organized **by the subject they iterate**, not
|
||||
by the concern they check. Cursor = row PK of that subject. Adding a
|
||||
new check = adding a per-row branch inside the job that walks that
|
||||
subject. See memory `project_consistency_jobs_landscape` for the full
|
||||
rationale + the merges/separations that fall out of the rule.
|
||||
|
||||
| Tenant | Iterates | Cursor | v1 checks | Notes |
|
||||
|---|---|---|---|---|
|
||||
| `drives_consistency` | `storage.drives` | drive UUID | `used_bytes` drift (drive + user envelope) | Shipped Slice 3. |
|
||||
| `folders_consistency` | `storage.folders` | folder UUID | `parent_trashed_mismatch` (live folder under trashed parent), `path_mismatch`, `lpath_mismatch` — both materialised columns compared to parent-chain reconstruction | Shipped Slice 4. Room to grow: `drive_id_parent_mismatch`, `orphan_root` (self-join already loads the fields). |
|
||||
| `files_consistency` | `storage.files` | file UUID | `parent_folder_trashed` (live file under trashed folder), `missing_blob` (severity `data_loss` — `blob_hash` present in neither `storage.blobs` nor `storage.chunk_manifests`), `chunk_missing` (severity `data_loss` — manifest exists but points at chunks absent from `storage.blobs`; typical dedup GC race), `blob_size_mismatch` (denormalised `files.size` diverges from the authoritative size — manifest first, blob fallback) | Shipped Slice 6, CDC-aware Slice 10. Handles both storage paths: `storage.chunk_manifests` (post-Apr-2026 FastCDC ingest, dominant path) and `storage.blobs` (pre-CDC whole-file blob, legacy fallback). Physical backend-existence checks (chunk bytes actually on disk) belong in `storage_consistency`. Room to grow: `drive_id_parent_mismatch`, mime-type reconciliation. |
|
||||
| `storage_consistency` | Storage backend (fs / S3) | object key / path | Each blob has a `storage.blobs` row (orphan detection) | `?deep=true` adds re-BLAKE3 + mime sniff. Orphan-side of the old bidirectional blob check + former `blob_integrity` + former `thumbnail_consistency`. |
|
||||
| `grants_consistency` (future) | `storage.role_grants` | grant UUID | subject/resource/granted_by exist | |
|
||||
| `storage_migration` | `storage.blobs` (source) → target backend | blob hash | Copy bytes; failures → `stats.failed_blobs` (and eventually `jobs.run_findings`) | Retires `Arc<RwLock<MigrationState>>` in `migration_job.rs`. |
|
||||
| `reextract_audio` | `storage.files` where audio | file UUID | Re-run audio-tag parser, upsert `audio_metadata` | Retires synchronous admin-request execution. |
|
||||
| `reextract_image` | `storage.files` where image/video | file UUID | Re-run EXIF/container date parser, upsert capture date | Same shape as reextract_audio. |
|
||||
| `consistency_batch` (wrapper) | Iterates registered `*_consistency` jobs | — (JobHandler, not RecoverableJobHandler) | Sequentially triggers each sub-job; `?deep=true` propagates | Shipped Slice 5. One-click "run all" without per-job clicks; exclusivity via `job_name` prevents concurrent batches from stepping on each other. Batch itself always returns `Ok` — child failures land in `outcome.extra.per_check[<name>].outcome`. |
|
||||
|
||||
**Not consistency**: `POST /api/admin/dedup/recalculate` is aggregate-
|
||||
stats-only (`unique_blobs`, `total_references`, `bytes_saved`) — one
|
||||
SELECT + one UPDATE. Kept as its own admin endpoint; do NOT fold into
|
||||
`storage_consistency` (different semantic — recompute vs verify).
|
||||
|
||||
### Verification (Part 2)
|
||||
|
||||
@@ -733,7 +743,7 @@ GET /api/admin/jobs/{name}/runs/{id}
|
||||
6. **Idempotent replay:** for consistency-check specifically, verify
|
||||
that re-processing the last unpersisted batch does NOT double-record
|
||||
findings (`UNIQUE (run_id, kind, resource_id)` on
|
||||
`admin.consistency_findings`).
|
||||
`jobs.run_findings`).
|
||||
7. **`RunOutcome` bridge log lines:** completed run logs
|
||||
`outcome=ok, extra.completed=true`; paused logs
|
||||
`outcome=ok, extra.paused=true`; failed logs `outcome=err, cause=handler`.
|
||||
@@ -790,26 +800,154 @@ endpoint returns a uniform `{ ok, outcome: JobOutcome }` envelope with
|
||||
job-specific fields under `outcome.extra`. Any external caller reading
|
||||
the old fields needs updating.
|
||||
|
||||
### Admin UI — /admin/jobs page (frontend, future slice)
|
||||
|
||||
Operators shouldn't have to `curl` these endpoints in production —
|
||||
they need a UI. Ships as a SvelteKit route once the backend surface is
|
||||
complete. Rough shape:
|
||||
|
||||
**Route:** `/admin/jobs` (SvelteKit page under `frontend/src/routes/admin/jobs/`).
|
||||
**Access:** admin-only; same guard as the rest of `/admin/*`.
|
||||
|
||||
**Page layout — one table, one drawer:**
|
||||
|
||||
```
|
||||
┌── Jobs ─────────────────────────────────────────────────────────────┐
|
||||
│ Name Cadence Last run Status Actions │
|
||||
│ ───────────────────────────────────────────────────────────────────│
|
||||
│ trash_cleanup every 24 h 3h ago ok [Run] │
|
||||
│ storage_reconcile every 10 m 4m ago ok [Run] │
|
||||
│ dedup_gc on-demand 1d ago ok [Run] │
|
||||
│ grant_cleanup every 24 h never — [Run] │
|
||||
│ drives_consistency on-demand never — [Run] │
|
||||
│ consistency_batch on-demand never — [Run] [Run deep] │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Row click opens a right-side drawer with:
|
||||
- Full JSON of the last outcome (`extra` fields explained per-job).
|
||||
- For recoverable jobs: run history table (`GET /jobs/{name}/runs`),
|
||||
each row expandable to full `RunSummary` (cursor, stats, params,
|
||||
error_message).
|
||||
- Per-run actions: `Cancel` (for Running rows only), `Trigger resume`
|
||||
(for Paused rows — same trigger endpoint, `run_or_resume` picks
|
||||
up the cursor).
|
||||
|
||||
**Data flow:**
|
||||
- `GET /api/admin/jobs` — populates the main table. Polled every 5 s
|
||||
when the page is visible (`document.visibilityState`).
|
||||
- `POST /api/admin/jobs/{name}/trigger` — the "Run" button. `deep=true`
|
||||
query for the "Run deep" variant (currently only shown on
|
||||
`consistency_batch`).
|
||||
- `POST /api/admin/jobs/{name}/cancel` — Cancel button on a Running
|
||||
recoverable run.
|
||||
- `GET /api/admin/jobs/{name}/runs` — populates the history table when
|
||||
the drawer opens.
|
||||
- `GET /api/admin/jobs/{name}/runs/{id}` — populates the per-run
|
||||
detail expander.
|
||||
|
||||
**No new backend endpoints required** — every screen is driven by
|
||||
what already exists.
|
||||
|
||||
**Visual conventions:**
|
||||
- Status colour: `ok` = green, `err` = red, `Running` = blue-pulse,
|
||||
`Paused` = amber, `CancelRequested` = amber-flash, `Completed` =
|
||||
neutral grey, `Failed` = red.
|
||||
- Findings surfacing is live as of Slice 7 (`jobs.run_findings` +
|
||||
`store.record_finding` + `GET /api/admin/jobs/{name}/runs/{id}/findings`).
|
||||
Drawer's "Findings" tab renders `kind`, `severity`, `resource_id`,
|
||||
and per-tenant `detail` JSON.
|
||||
|
||||
**Slice ordering:** frontend page is a follow-up PR, not blocking any
|
||||
backend slice. Order of appearance:
|
||||
1. Backend Part 2 slices (engine, admin surface, first tenant) — done.
|
||||
2. `jobs.run_findings` table + `store.record_finding` API — done (Slice 7).
|
||||
3. `consistency_batch` + more tenants — done (Slices 5–6: drives + folders + files, plus batch).
|
||||
4. Frontend `/admin/jobs` page — takes the completed backend surface
|
||||
as-is; no backend changes required by the UI landing.
|
||||
5. Progress estimation on `RunSummary.progress` (`fraction`, `kind`,
|
||||
`scanned`, `total`) — **done (Slice 9)**. Tenants that CAN count
|
||||
their subject override `RecoverableJobHandler::count_total()`;
|
||||
`run_or_resume` seeds `params.total_rows` + `params.progress_kind`
|
||||
on fresh runs; `row_to_summary` derives the `progress` block at
|
||||
serialisation time. UI renders a bar; `kind = "approximate"` runs
|
||||
get a striped fill so operators recognise proxy-derived
|
||||
estimates. See memory `project_job_progress_estimation`.
|
||||
|
||||
### Notifications & alerting
|
||||
|
||||
Silent failure is the enemy — a consistency check that finds a
|
||||
data-loss finding at 3 AM Sunday should reach an operator, not sit in
|
||||
the log stream unread. When SMTP is wired, the supervisor emits an
|
||||
alert email on the following:
|
||||
|
||||
- **Any job dispatch returns `JobOutcome::Err`.** Applies to both
|
||||
Part 1 handler errors and Part 2 recoverable `RunOutcome::Failed`
|
||||
(which translates to `Err` via `run_or_resume`'s bridge). Subject
|
||||
line: `[OxiCloud] Job <name> failed`. Body includes: job name,
|
||||
cause (`handler|timeout|panicked`), error message, run_id (Part 2
|
||||
only), elapsed_ms, log-timestamp for grep, link to
|
||||
`/admin/jobs?highlight=<name>` when the UI lands.
|
||||
- **Consistency check surfaces one or more findings** (deferred to
|
||||
the `jobs.run_findings` migration). Applies only to `*_consistency`
|
||||
tenants. Body includes: run_id, findings count grouped by
|
||||
`(kind, severity)`, worst-severity example, link to
|
||||
`/admin/jobs/{name}/runs/{id}` when the UI lands.
|
||||
|
||||
**Delivery conditions:**
|
||||
- Silent no-op when `email_sender` on `AppState` is `None` (SMTP not
|
||||
configured). No error, no log spam — the mechanism is opt-in
|
||||
through SMTP presence.
|
||||
- Recipient: every user with `role = 'admin'`. Not a hardcoded
|
||||
address — same rule as any admin-scoped notification the codebase
|
||||
already sends.
|
||||
- Rate limit: **at most 1 email per (job_name, kind) per 6 hours**,
|
||||
keyed off an in-memory dedup table on `AppState`. Prevents a
|
||||
flapping job (fails, retries, fails, ...) from mailbombing.
|
||||
6 h chosen to match the operator-attention interval — a real
|
||||
ongoing failure gets 4 alerts/day, enough to be noticed, not
|
||||
enough to be filtered.
|
||||
- Configurable OFF per job via env: `OXICLOUD_JOB_<NAME>_ALERT_ON_FAIL=false`
|
||||
(default `true`). Same shape as the existing enable/disable knobs.
|
||||
|
||||
**Implementation notes** (for whichever slice picks this up):
|
||||
- Reuses `EmailSender` port + `MagicLinkInviteService`-style templating
|
||||
under `askama`. New template files:
|
||||
`templates/emails/job_failed.{html,txt}` and
|
||||
`templates/emails/consistency_findings.{html,txt}`.
|
||||
- Dedup table lives on `AppState.job_alert_dedup:
|
||||
Arc<Mutex<HashMap<(String, String), Instant>>>`. Cleaned lazily on
|
||||
insert.
|
||||
- Called from `SchedulerEngine::log_outcome` (Part 1 path) and from
|
||||
`run_or_resume`'s terminal-write branch (Part 2 path). Both already
|
||||
see the `JobOutcome`; adding a fire-and-forget email dispatch is
|
||||
~10 lines each.
|
||||
|
||||
**Scope-out:** no Slack / webhook / PagerDuty integration in v1.
|
||||
Email is the ONE alert channel until an operator concretely asks for
|
||||
another. Layering webhooks on top later is trivial — same
|
||||
"terminal outcome → notification" hook, different sink.
|
||||
|
||||
### Config surface — env vars
|
||||
|
||||
Canonical form for every job (Part 1 or Part 2 alike, AND for core
|
||||
workers even though they don't register with the scheduler):
|
||||
**No new convention.** Each service keeps its natural per-service
|
||||
prefix (`OXICLOUD_GRANT_CLEANUP_*`, `OXICLOUD_STORAGE_USAGE_*`, …).
|
||||
The `GET /api/admin/jobs` endpoint already gives operators a runtime
|
||||
view of every registered job's interval, so grepping env-var prefixes
|
||||
is no longer the primary discovery path.
|
||||
|
||||
```
|
||||
OXICLOUD_JOB_<NAME>_ENABLED
|
||||
OXICLOUD_JOB_<NAME>_INTERVAL_HOURS # or _INTERVAL_SECS for sub-hour cadences
|
||||
OXICLOUD_JOB_<NAME>_<CUSTOM>... # e.g. _GRACE_HOURS, _BATCH_SIZE
|
||||
```
|
||||
Earlier drafts proposed a uniform `OXICLOUD_JOB_<NAME>_INTERVAL_*`
|
||||
convention, with legacy names as warned aliases. Killed 2026-07-28
|
||||
(Ed): normalising only the interval knob while leaving domain-specific
|
||||
tunables (`GRACE_DAYS`, `BATCH_SIZE`, …) at the natural prefix creates
|
||||
*intra-service* prefix drift — worse than the *cross-service* drift it
|
||||
was meant to solve. A service either goes fully to `OXICLOUD_JOB_*`
|
||||
(disruptive rename of every knob) or fully stays at its native prefix
|
||||
(no rename). We stay.
|
||||
|
||||
Core workers reuse this naming purely for uniform operator ergonomics
|
||||
(e.g. `OXICLOUD_JOB_TREE_ETAG_FLUSH_INTERVAL_MS`) — the convention is
|
||||
what operators grep for; whether the loop is scheduler-driven or a
|
||||
dedicated `tokio::spawn` is an implementation detail they don't see.
|
||||
|
||||
Existing per-service env vars keep working as **aliases** during
|
||||
migration — `OXICLOUD_GRANT_CLEANUP_INTERVAL_HOURS` reads first, falls
|
||||
back to `OXICLOUD_JOB_GRANT_CLEANUP_INTERVAL_HOURS`. Deprecated aliases
|
||||
warn once on startup and stay recognised through one minor version.
|
||||
The one real gap is **trash_cleanup has no env var today** (hardcoded
|
||||
24h in DI). Adding `OXICLOUD_TRASH_CLEANUP_INTERVAL_HOURS` when we
|
||||
need it uses the natural prefix — no new convention needed.
|
||||
|
||||
### Logging schema
|
||||
|
||||
@@ -874,7 +1012,7 @@ precludes it.
|
||||
|
||||
### Job-history observability
|
||||
|
||||
`admin.background_runs` already carries the latest run per Part 2 job
|
||||
`jobs.recoverable_runs` already carries the latest run per Part 2 job
|
||||
— "last run time + status" is a `SELECT DISTINCT ON (job_name) …`
|
||||
query. Deeper history (retention window, per-run drill-down UI) is
|
||||
deferred; the log stream is the source of truth for older runs.
|
||||
@@ -889,7 +1027,7 @@ No such need today.
|
||||
|
||||
- **Cross-job dependencies.** Register-time ordering only, not runtime
|
||||
graph.
|
||||
- **Retention pruning of terminal `background_runs` rows.** Deferred
|
||||
- **Retention pruning of terminal `recoverable_runs` rows.** Deferred
|
||||
until the volume warrants a policy.
|
||||
- **Prometheus / OpenMetrics export.** Log-only for now.
|
||||
- **Distributed scheduling.** Single-process. If OxiCloud ever runs
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Admin JobRegistry endpoints — `/api/admin/jobs*` (see
|
||||
* `docs/plan/job-registry.md`). Powers the "Jobs" tab of the admin panel.
|
||||
*
|
||||
* Every mutation goes through the standard admin auth path (Bearer JWT
|
||||
* + admin-middleware role check). Read endpoints are cheap enough to
|
||||
* poll while the panel is open.
|
||||
*/
|
||||
import { apiFetch, apiJson } from '$lib/api/client';
|
||||
import { getCsrfHeaders } from '$lib/api/csrf';
|
||||
import type { Finding, JobOutcome, JobSummary, RunSummary } from '$lib/api/types';
|
||||
|
||||
const JSON_HEADERS = { 'Content-Type': 'application/json' };
|
||||
|
||||
/**
|
||||
* Envelope wrapping the outcome from `POST /api/admin/jobs/{name}/trigger`.
|
||||
* `ok: true` means "dispatch reached the handler"; the handler's own
|
||||
* pass/fail is in `outcome.outcome`. For `consistency_batch`, per-child
|
||||
* outcomes are inside `outcome.extra.per_check`.
|
||||
*/
|
||||
export interface TriggerResponse {
|
||||
ok: boolean;
|
||||
outcome: JobOutcome;
|
||||
}
|
||||
|
||||
/** Envelope from `POST /api/admin/jobs/{name}/cancel`. `run_id` is
|
||||
* the id of the run whose `Running` status was flipped to
|
||||
* `CancelRequested` (null when nothing was in flight to cancel). */
|
||||
export interface CancelResponse {
|
||||
ok: boolean;
|
||||
run_id: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* `GET /api/admin/jobs` — full registry snapshot. One row per registered
|
||||
* job (periodic + recoverable + coordinators like `consistency_batch`,
|
||||
* which register as plain JobHandlers).
|
||||
*/
|
||||
export function listJobs(): Promise<JobSummary[]> {
|
||||
return apiJson<JobSummary[]>('/api/admin/jobs', { credentials: 'same-origin' });
|
||||
}
|
||||
|
||||
/**
|
||||
* `POST /api/admin/jobs/{name}/trigger?force=X&deep=X` — dispatch a job
|
||||
* on-demand. `force` bypasses per-tenant idempotency checks (e.g.
|
||||
* `trash_cleanup` skipping when nothing is due). `deep` opts into slow
|
||||
* variants (currently only `storage_consistency`, propagated by
|
||||
* `consistency_batch` to every child).
|
||||
*
|
||||
* Throws on 4xx / 5xx with the backend's error message when present.
|
||||
* A 404 means the job name isn't registered — surface that specifically
|
||||
* so callers can distinguish "typo" from "handler blew up".
|
||||
*/
|
||||
export async function triggerJob(
|
||||
name: string,
|
||||
opts: { force?: boolean; deep?: boolean } = {}
|
||||
): Promise<TriggerResponse> {
|
||||
const params = new URLSearchParams();
|
||||
if (opts.force) params.set('force', 'true');
|
||||
if (opts.deep) params.set('deep', 'true');
|
||||
const q = params.toString();
|
||||
const url = `/api/admin/jobs/${encodeURIComponent(name)}/trigger${q ? `?${q}` : ''}`;
|
||||
const res = await apiFetch(url, {
|
||||
method: 'POST',
|
||||
credentials: 'same-origin',
|
||||
headers: { ...JSON_HEADERS, ...getCsrfHeaders() }
|
||||
});
|
||||
if (!res.ok) {
|
||||
let msg = `trigger failed: ${res.status}`;
|
||||
try {
|
||||
const body = (await res.json()) as { error?: string; message?: string };
|
||||
msg = body.error ?? body.message ?? msg;
|
||||
} catch {
|
||||
/* no JSON body */
|
||||
}
|
||||
throw new Error(msg);
|
||||
}
|
||||
return (await res.json()) as TriggerResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* `POST /api/admin/jobs/{name}/cancel` — cooperatively request cancel
|
||||
* of the currently running instance. The handler observes it on its
|
||||
* next `store.status()` poll and returns `RunOutcome::Paused` at the
|
||||
* next safe boundary. If nothing is running, this is a no-op that
|
||||
* returns `run_id: null`.
|
||||
*/
|
||||
export async function cancelJob(name: string): Promise<CancelResponse> {
|
||||
const res = await apiFetch(`/api/admin/jobs/${encodeURIComponent(name)}/cancel`, {
|
||||
method: 'POST',
|
||||
credentials: 'same-origin',
|
||||
headers: { ...JSON_HEADERS, ...getCsrfHeaders() }
|
||||
});
|
||||
if (!res.ok) {
|
||||
let msg = `cancel failed: ${res.status}`;
|
||||
try {
|
||||
const body = (await res.json()) as { error?: string; message?: string };
|
||||
msg = body.error ?? body.message ?? msg;
|
||||
} catch {
|
||||
/* no JSON body */
|
||||
}
|
||||
throw new Error(msg);
|
||||
}
|
||||
return (await res.json()) as CancelResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable
|
||||
* runs for `name`, newest first. Backend caps `limit` at 100.
|
||||
*/
|
||||
export function listRuns(name: string, limit = 20): Promise<RunSummary[]> {
|
||||
return apiJson<RunSummary[]>(`/api/admin/jobs/${encodeURIComponent(name)}/runs?limit=${limit}`, {
|
||||
credentials: 'same-origin'
|
||||
});
|
||||
}
|
||||
|
||||
/** Envelope from `POST /api/admin/jobs/runs/purge`. `purged` is
|
||||
* the count of terminal-run rows deleted (findings cascade with
|
||||
* their parent run via the FK, no separate counter). */
|
||||
export interface PurgeResponse {
|
||||
purged: number;
|
||||
retention_days: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* `POST /api/admin/jobs/runs/purge?days=N` — operator-triggered
|
||||
* retention cleanup. Deletes terminal runs (`Completed`, `Failed`)
|
||||
* with `completed_at` older than `days` days ago; associated
|
||||
* `jobs.run_findings` rows drop with them via CASCADE. Non-terminal
|
||||
* runs (`Running`, `Paused`, `CancelRequested`) are ALWAYS
|
||||
* preserved regardless of age.
|
||||
*
|
||||
* Backend enforces a minimum of 1 day defensively.
|
||||
*/
|
||||
export async function purgeJobRuns(days = 30): Promise<PurgeResponse> {
|
||||
const res = await apiFetch(`/api/admin/jobs/runs/purge?days=${days}`, {
|
||||
method: 'POST',
|
||||
credentials: 'same-origin',
|
||||
headers: { ...JSON_HEADERS, ...getCsrfHeaders() }
|
||||
});
|
||||
if (!res.ok) {
|
||||
let msg = `purge failed: ${res.status}`;
|
||||
try {
|
||||
const body = (await res.json()) as { error?: string; message?: string };
|
||||
msg = body.error ?? body.message ?? msg;
|
||||
} catch {
|
||||
/* no JSON body */
|
||||
}
|
||||
throw new Error(msg);
|
||||
}
|
||||
return (await res.json()) as PurgeResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* `GET /api/admin/jobs/{name}/runs/{id}/findings?limit=N&offset=M` —
|
||||
* paginated findings for a specific run. Empty list = clean run,
|
||||
* 404 = unknown run id.
|
||||
*/
|
||||
export function listFindings(
|
||||
name: string,
|
||||
runId: string,
|
||||
opts: { limit?: number; offset?: number } = {}
|
||||
): Promise<Finding[]> {
|
||||
const params = new URLSearchParams();
|
||||
params.set('limit', String(opts.limit ?? 100));
|
||||
if (opts.offset) params.set('offset', String(opts.offset));
|
||||
return apiJson<Finding[]>(
|
||||
`/api/admin/jobs/${encodeURIComponent(name)}/runs/${encodeURIComponent(runId)}/findings?${params}`,
|
||||
{ credentials: 'same-origin' }
|
||||
);
|
||||
}
|
||||
@@ -522,3 +522,102 @@ export interface FolderAncestorsResponse {
|
||||
ancestors: FolderAncestor[];
|
||||
access_source: AccessSource;
|
||||
}
|
||||
|
||||
// ─── Job registry (Part 1 + Part 2) ────────────────────────────────────────
|
||||
//
|
||||
// Maps `src/infrastructure/scheduler/*` DTOs 1:1. See
|
||||
// `docs/plan/job-registry.md` for the backend contract; the shapes below
|
||||
// are what the `/api/admin/jobs*` endpoints emit.
|
||||
|
||||
/**
|
||||
* `JobOutcome` — the uniform outcome the scheduler logs and stores for
|
||||
* every job dispatch. Serialised with `#[serde(tag = "outcome")]` so the
|
||||
* discriminant is the `outcome` field, not the object key.
|
||||
*/
|
||||
export type JobOutcome =
|
||||
| { outcome: 'ok'; count: number; extra?: unknown }
|
||||
| { outcome: 'err'; message: string };
|
||||
|
||||
/**
|
||||
* `JobSummary` — one row per registered job in `GET /api/admin/jobs`.
|
||||
* Cadence + last-run bookkeeping. `interval_ms` / `next_run_at` are
|
||||
* `undefined` on on-demand jobs (serde skips `Option::None`).
|
||||
*/
|
||||
export interface JobSummary {
|
||||
name: string;
|
||||
interval_ms?: number;
|
||||
next_run_at?: string;
|
||||
last_run_at?: string;
|
||||
last_outcome?: JobOutcome;
|
||||
running: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* `RunStatus` values allowed in `jobs.recoverable_runs.status`. The
|
||||
* non-terminal set (Running / Paused / CancelRequested) is what the
|
||||
* DB's `one_active_run_per_job` partial unique index scopes.
|
||||
*/
|
||||
export type RunStatus = 'Running' | 'Paused' | 'CancelRequested' | 'Completed' | 'Failed';
|
||||
|
||||
/**
|
||||
* `RunSummary` — one row per recoverable-job run from
|
||||
* `GET /api/admin/jobs/{name}/runs`. Terminal + non-terminal rows both
|
||||
* appear. `stats` / `params` are opaque JSON — job-specific shape;
|
||||
* consumers should key off `job_name` to decide what to render.
|
||||
* `cursor_hex` is present only when the run has advanced past the
|
||||
* initial state (paused mid-scan is the typical case).
|
||||
*/
|
||||
export interface RunSummary {
|
||||
id: string;
|
||||
job_name: string;
|
||||
status: RunStatus;
|
||||
started_at: string;
|
||||
last_progress_at: string;
|
||||
completed_at?: string;
|
||||
stats: Record<string, unknown>;
|
||||
params: Record<string, unknown>;
|
||||
cursor_hex?: string;
|
||||
error_message?: string;
|
||||
/** Populated when the tenant reported a countable subject at run
|
||||
* start (`RecoverableJobHandler::count_total`). Absent when the
|
||||
* tenant can't count — the UI hides the progress bar and falls
|
||||
* back to raw `scanned_count`. */
|
||||
progress?: RunProgress;
|
||||
}
|
||||
|
||||
/**
|
||||
* Confidence level of a `RunProgress` fraction. Wire lowercase per
|
||||
* the `#[serde(rename_all = "lowercase")]` on the Rust enum.
|
||||
*
|
||||
* - `count` — `scanned_count / total_rows` where `total_rows` came
|
||||
* from a definitive `COUNT(*)` on the subject table.
|
||||
* - `approximate` — proxy-derived total (e.g. `storage_consistency`
|
||||
* using DB blob count as a stand-in for backend object count).
|
||||
* Fraction can legitimately exceed 1.0 at run end — the deviation
|
||||
* quantifies the drift the check is looking for.
|
||||
*/
|
||||
export type ProgressKind = 'count' | 'approximate';
|
||||
|
||||
export interface RunProgress {
|
||||
fraction: number;
|
||||
kind: ProgressKind;
|
||||
scanned: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* `Finding` — one row from `GET /api/admin/jobs/{name}/runs/{id}/findings`.
|
||||
* Persisted by consistency tenants via `store.record_finding()`. Consumers
|
||||
* key off `kind` to know the shape of `detail` (per-tenant JSON — e.g.
|
||||
* `stale_used_bytes` carries `{cached, actual, delta}`; `missing_blob`
|
||||
* carries `{blob_hash}`; …).
|
||||
*/
|
||||
export interface Finding {
|
||||
id: string;
|
||||
run_id: string;
|
||||
kind: string;
|
||||
severity: string;
|
||||
resource_id?: string;
|
||||
detail: Record<string, unknown>;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -32,15 +32,12 @@
|
||||
const palette = lazyComponent(() => import('$lib/components/CommandPalette.svelte'));
|
||||
|
||||
interface NavLink {
|
||||
href:
|
||||
| '/files'
|
||||
| '/shared'
|
||||
| '/shared-with-me'
|
||||
| '/recent'
|
||||
| '/favorites'
|
||||
| '/photos'
|
||||
| '/music'
|
||||
| '/trash';
|
||||
/**
|
||||
* String rather than a literal union so admin links (which
|
||||
* include a dynamic path segment) can share the same shape.
|
||||
* `resolve()` accepts any string, so no type-level cost.
|
||||
*/
|
||||
href: string;
|
||||
label: string;
|
||||
icon: string;
|
||||
/** Stable key driving the per-section icon colour (see sidebar.css). */
|
||||
@@ -69,12 +66,103 @@
|
||||
{ href: '/trash', label: t('nav.trash', 'Trash'), icon: 'trash', section: 'trash' }
|
||||
];
|
||||
|
||||
// Admin sidebar — populated when the URL is under /admin. The
|
||||
// admin +page.svelte used to render its own horizontal tab
|
||||
// strip; that was displaced here so the section navigation
|
||||
// scales past ~7 items and matches deep-link URLs from the
|
||||
// address bar.
|
||||
const ADMIN_LINKS: NavLink[] = [
|
||||
{
|
||||
href: '/admin',
|
||||
label: t('admin.dashboard', 'Dashboard'),
|
||||
icon: 'chart-pie',
|
||||
section: 'admin-dashboard'
|
||||
},
|
||||
{
|
||||
href: '/admin/users',
|
||||
label: t('admin.users', 'Users'),
|
||||
icon: 'users',
|
||||
section: 'admin-users'
|
||||
},
|
||||
{
|
||||
href: '/admin/drives',
|
||||
label: t('admin.drives', 'Drives'),
|
||||
icon: 'folder',
|
||||
section: 'admin-drives'
|
||||
},
|
||||
{
|
||||
href: '/admin/mounts',
|
||||
label: t('admin.mounts', 'External Mounts'),
|
||||
icon: 'folder',
|
||||
section: 'admin-mounts'
|
||||
},
|
||||
{
|
||||
href: '/admin/oidc',
|
||||
label: t('admin.oidc', 'OIDC / SSO'),
|
||||
icon: 'key',
|
||||
section: 'admin-oidc'
|
||||
},
|
||||
{
|
||||
href: '/admin/storage',
|
||||
label: t('admin.storage_tab', 'Storage'),
|
||||
icon: 'database',
|
||||
section: 'admin-storage'
|
||||
},
|
||||
{
|
||||
href: '/admin/smtp',
|
||||
label: t('admin.smtp', 'Email (SMTP)'),
|
||||
icon: 'envelope',
|
||||
section: 'admin-smtp'
|
||||
},
|
||||
{
|
||||
href: '/admin/plugins',
|
||||
label: t('admin.plugins', 'Plugins'),
|
||||
icon: 'layer-group',
|
||||
section: 'admin-plugins'
|
||||
},
|
||||
{
|
||||
href: '/admin/jobs',
|
||||
label: t('admin.jobs.tab', 'Jobs'),
|
||||
icon: 'cogs',
|
||||
section: 'admin-jobs'
|
||||
}
|
||||
];
|
||||
|
||||
const isAdmin = $derived(session.user?.role === 'admin');
|
||||
|
||||
// Any URL under /admin swaps the sidebar to admin mode. Uses
|
||||
// startsWith so a trailing slash / query params / hash don't
|
||||
// desync. Root `/admin` counts too (dashboard).
|
||||
const isAdminSection = $derived(page.url.pathname.startsWith('/admin'));
|
||||
const currentLinks = $derived(isAdminSection ? ADMIN_LINKS : LINKS);
|
||||
|
||||
function active(href: string): boolean {
|
||||
return page.url.pathname === href || page.url.pathname.startsWith(`${href}/`);
|
||||
}
|
||||
|
||||
// Sidebar-item active check. Non-admin links use `active()`
|
||||
// (matches href + any subpath). Admin links need a stricter
|
||||
// rule for `/admin` itself — a plain `startsWith('/admin/')`
|
||||
// would light up the Dashboard item on `/admin/drives` too.
|
||||
// So `/admin` matches ONLY the exact path; every other admin
|
||||
// item uses the same startsWith rule as before.
|
||||
function activeLink(href: string): boolean {
|
||||
if (href === '/admin') return page.url.pathname === '/admin';
|
||||
return active(href);
|
||||
}
|
||||
|
||||
/**
|
||||
* Data-driven sidebar links (`LINKS`, `ADMIN_LINKS`) hold
|
||||
* runtime strings, not compile-time route keys. SvelteKit's
|
||||
* typed `resolve()` refuses them; we know they're valid
|
||||
* routes at runtime. Cast at the callsite, one place, so
|
||||
* the template stays clean.
|
||||
*/
|
||||
function navHref(href: string): string {
|
||||
// @ts-expect-error runtime-known route string, not a literal typed key
|
||||
return resolve(href);
|
||||
}
|
||||
|
||||
// ── Sidebar drop targets ─────────────────────────────────────────────────
|
||||
// The row-drag on `/files` (and other resource surfaces) sets a
|
||||
// `application/x-oxi-item` MIME with a JSON array of `{ id, name, kind }`.
|
||||
@@ -467,16 +555,50 @@
|
||||
<div class="app-name">OxiCloud</div>
|
||||
</a>
|
||||
|
||||
<nav class="nav-menu" aria-label={t('nav.primary', 'Primary')}>
|
||||
{#each LINKS as link (link.href)}
|
||||
{@const isDropTarget = link.href === '/favorites' || link.href === '/trash'}
|
||||
<nav
|
||||
class="nav-menu"
|
||||
class:nav-menu--admin={isAdminSection}
|
||||
aria-label={isAdminSection
|
||||
? t('nav.admin_sections', 'Admin sections')
|
||||
: t('nav.primary', 'Primary')}
|
||||
>
|
||||
{#if isAdminSection}
|
||||
<!--
|
||||
"Back to app" escape hatch — the sidebar swaps to
|
||||
admin mode when the URL is under /admin/*, so the
|
||||
usual Files/Shared links disappear. A dedicated back
|
||||
entry gives the operator a one-click way out without
|
||||
having to hunt through the user menu.
|
||||
-->
|
||||
<a
|
||||
class="nav-item nav-item--back"
|
||||
href={resolve('/files')}
|
||||
data-testid="appshell-nav-admin-back-link"
|
||||
onclick={() => (sidebarOpen = false)}
|
||||
>
|
||||
<Icon name="arrow-left" />
|
||||
<span>{t('nav.back_to_app', 'Back to OxiCloud')}</span>
|
||||
</a>
|
||||
{/if}
|
||||
<!--
|
||||
Sidebar links come from data-driven arrays (LINKS,
|
||||
ADMIN_LINKS), so hrefs are runtime strings rather than
|
||||
compile-time-known route keys. `navHref()` internally
|
||||
routes them through `resolve()`, but ESLint's static
|
||||
check can't see through the wrapper. Disabling the
|
||||
rule here rather than sprinkling per-line directives.
|
||||
-->
|
||||
<!-- eslint-disable svelte/no-navigation-without-resolve -->
|
||||
{#each currentLinks as link (link.href)}
|
||||
{@const isDropTarget =
|
||||
!isAdminSection && (link.href === '/favorites' || link.href === '/trash')}
|
||||
<a
|
||||
class="nav-item"
|
||||
class:active={active(link.href)}
|
||||
class:active={activeLink(link.href)}
|
||||
class:nav-item--drop-target={isDropTarget && sidebarDropHref === link.href}
|
||||
href={resolve(link.href)}
|
||||
href={navHref(link.href)}
|
||||
data-section={link.section}
|
||||
data-testid={`appshell-nav-${link.href.replace(/^\//, '')}-link`}
|
||||
data-testid={`appshell-nav-${link.section}-link`}
|
||||
onclick={() => (sidebarOpen = false)}
|
||||
ondragover={isDropTarget ? (e) => sidebarOnDragOver(e, link.href) : undefined}
|
||||
ondragleave={isDropTarget ? () => sidebarOnDragLeave(link.href) : undefined}
|
||||
@@ -485,10 +607,11 @@
|
||||
<Icon name={link.icon} />
|
||||
<span>{link.label}</span>
|
||||
</a>
|
||||
{#if link.href === '/files' && !session.isExternalUser}
|
||||
{#if !isAdminSection && link.href === '/files' && !session.isExternalUser}
|
||||
<DrivePicker onnavigate={() => (sidebarOpen = false)} />
|
||||
{/if}
|
||||
{/each}
|
||||
<!-- eslint-enable svelte/no-navigation-without-resolve -->
|
||||
</nav>
|
||||
|
||||
{#if session.user}
|
||||
|
||||
+85
-100
@@ -1,4 +1,5 @@
|
||||
<script lang="ts">
|
||||
import { page } from '$app/state';
|
||||
import { errorMessage, errorToast } from '$lib/utils/errors';
|
||||
import { dateTimeFormatFor } from '$lib/utils/display';
|
||||
import {
|
||||
@@ -76,6 +77,7 @@
|
||||
DrivePoliciesPartial,
|
||||
User
|
||||
} from '$lib/api/types';
|
||||
import AdminJobsPanel from '$lib/components/AdminJobsPanel.svelte';
|
||||
import Icon from '$lib/icons/Icon.svelte';
|
||||
import Modal from '$lib/components/Modal.svelte';
|
||||
import OwnerAvatarStack from '$lib/components/OwnerAvatarStack.svelte';
|
||||
@@ -172,8 +174,69 @@
|
||||
}
|
||||
}
|
||||
|
||||
type Tab = 'dashboard' | 'users' | 'drives' | 'mounts' | 'plugins' | 'oidc' | 'storage' | 'smtp';
|
||||
let tab = $state<Tab>('dashboard');
|
||||
type Tab =
|
||||
| 'dashboard'
|
||||
| 'users'
|
||||
| 'drives'
|
||||
| 'mounts'
|
||||
| 'plugins'
|
||||
| 'oidc'
|
||||
| 'storage'
|
||||
| 'smtp'
|
||||
| 'jobs';
|
||||
|
||||
const VALID_TABS: readonly Tab[] = [
|
||||
'dashboard',
|
||||
'users',
|
||||
'drives',
|
||||
'mounts',
|
||||
'plugins',
|
||||
'oidc',
|
||||
'storage',
|
||||
'smtp',
|
||||
'jobs'
|
||||
];
|
||||
|
||||
function parseTab(raw: string | undefined): Tab {
|
||||
return VALID_TABS.includes(raw as Tab) ? (raw as Tab) : 'dashboard';
|
||||
}
|
||||
|
||||
// URL is the source of truth for the current tab. Navigation
|
||||
// is via the AppShell sidebar (see `ADMIN_LINKS` there);
|
||||
// this page just reads `page.params.tab` and renders the
|
||||
// matching content block. Unidirectional URL→state means no
|
||||
// `$effect` loop is even possible.
|
||||
const tab = $derived<Tab>(parseTab(page.params.tab));
|
||||
|
||||
/**
|
||||
* Human-readable label for the current section — feeds the
|
||||
* page title (`Admin › Jobs · OxiCloud`) and the h1. Kept in
|
||||
* sync with the `ADMIN_LINKS` labels in AppShell manually
|
||||
* (small list, unlikely to drift). Extracting to a shared
|
||||
* module would be over-engineering for 9 strings.
|
||||
*/
|
||||
const tabLabel = $derived.by<string>(() => {
|
||||
switch (tab) {
|
||||
case 'dashboard':
|
||||
return t('admin.dashboard', 'Dashboard');
|
||||
case 'users':
|
||||
return t('admin.users', 'Users');
|
||||
case 'drives':
|
||||
return t('admin.drives', 'Drives');
|
||||
case 'mounts':
|
||||
return t('admin.mounts', 'External Mounts');
|
||||
case 'plugins':
|
||||
return t('admin.plugins', 'Plugins');
|
||||
case 'oidc':
|
||||
return t('admin.oidc', 'OIDC / SSO');
|
||||
case 'storage':
|
||||
return t('admin.storage_tab', 'Storage');
|
||||
case 'smtp':
|
||||
return t('admin.smtp', 'Email (SMTP)');
|
||||
case 'jobs':
|
||||
return t('admin.jobs.tab', 'Jobs');
|
||||
}
|
||||
});
|
||||
|
||||
// Dashboard
|
||||
let dashboard = $state<AdminDashboard | null>(null);
|
||||
@@ -1462,7 +1525,8 @@
|
||||
plugins: false,
|
||||
oidc: false,
|
||||
storage: false,
|
||||
smtp: false
|
||||
smtp: false,
|
||||
jobs: false
|
||||
});
|
||||
|
||||
$effect(() => {
|
||||
@@ -1493,7 +1557,9 @@
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:head><title>{t('admin.title', 'Admin')} · OxiCloud</title></svelte:head>
|
||||
<svelte:head>
|
||||
<title>{t('admin.title', 'Admin')} › {tabLabel} · OxiCloud</title>
|
||||
</svelte:head>
|
||||
|
||||
{#snippet envBadge(on: boolean)}
|
||||
{#if on}
|
||||
@@ -1503,83 +1569,20 @@
|
||||
{/if}
|
||||
{/snippet}
|
||||
|
||||
<!--
|
||||
The horizontal tab-bar that used to live here was displaced into
|
||||
the shared AppShell sidebar (context-aware — swaps to admin
|
||||
sections when the URL is under /admin/*). This page now renders
|
||||
ONLY the tab content; navigation is via the sidebar + URL. See
|
||||
`lib/components/AppShell.svelte` (ADMIN_LINKS).
|
||||
-->
|
||||
<main class="admin">
|
||||
<h1>{t('admin.title', 'Admin')}</h1>
|
||||
|
||||
<div class="tabs" role="tablist">
|
||||
<button
|
||||
role="tab"
|
||||
data-testid="admin-dashboard-tab"
|
||||
aria-selected={tab === 'dashboard'}
|
||||
onclick={() => (tab = 'dashboard')}
|
||||
>
|
||||
<Icon name="chart-pie" />
|
||||
{t('admin.dashboard', 'Dashboard')}
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
data-testid="admin-users-tab"
|
||||
aria-selected={tab === 'users'}
|
||||
onclick={() => (tab = 'users')}
|
||||
>
|
||||
<Icon name="users" />
|
||||
{t('admin.users', 'Users')}
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
data-testid="admin-drives-tab"
|
||||
aria-selected={tab === 'drives'}
|
||||
onclick={() => (tab = 'drives')}
|
||||
>
|
||||
<Icon name="folder" />
|
||||
{t('admin.drives', 'Drives')}
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
data-testid="admin-mounts-tab"
|
||||
aria-selected={tab === 'mounts'}
|
||||
onclick={() => (tab = 'mounts')}
|
||||
>
|
||||
<Icon name="folder" />
|
||||
{t('admin.mounts', 'External Mounts')}
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
data-testid="admin-oidc-tab"
|
||||
aria-selected={tab === 'oidc'}
|
||||
onclick={() => (tab = 'oidc')}
|
||||
>
|
||||
<Icon name="key" />
|
||||
{t('admin.oidc', 'OIDC / SSO')}
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
data-testid="admin-storage-tab"
|
||||
aria-selected={tab === 'storage'}
|
||||
onclick={() => (tab = 'storage')}
|
||||
>
|
||||
<Icon name="database" />
|
||||
{t('admin.storage_tab', 'Storage')}
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
data-testid="admin-smtp-tab"
|
||||
aria-selected={tab === 'smtp'}
|
||||
onclick={() => (tab = 'smtp')}
|
||||
>
|
||||
<Icon name="envelope" />
|
||||
{t('admin.smtp', 'Email (SMTP)')}
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
data-testid="admin-plugins-tab"
|
||||
aria-selected={tab === 'plugins'}
|
||||
onclick={() => (tab = 'plugins')}
|
||||
>
|
||||
<Icon name="layer-group" />
|
||||
{t('admin.plugins', 'Plugins')}
|
||||
</button>
|
||||
</div>
|
||||
<!--
|
||||
H1 shows the current section since the sidebar is what
|
||||
communicates which admin area we're in — the plain "Admin"
|
||||
h1 was informationless once the tab bar moved out.
|
||||
-->
|
||||
<h1>{tabLabel}</h1>
|
||||
|
||||
{#if tab === 'dashboard'}
|
||||
{#if dashboardError}
|
||||
@@ -2787,6 +2790,8 @@
|
||||
</tbody>
|
||||
</table>
|
||||
{/if}
|
||||
{:else if tab === 'jobs'}
|
||||
<AdminJobsPanel />
|
||||
{:else if !pluginsAvailable}
|
||||
<p class="status">{t('admin.plugins_disabled', 'The plugin subsystem is disabled.')}</p>
|
||||
{:else if pluginsError}
|
||||
@@ -4092,26 +4097,6 @@
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.tabs {
|
||||
display: flex;
|
||||
gap: 0.25rem;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.tabs button {
|
||||
padding: 0.5rem 1rem;
|
||||
border: none;
|
||||
background: none;
|
||||
color: var(--color-text-muted);
|
||||
cursor: pointer;
|
||||
border-bottom: 2px solid transparent;
|
||||
}
|
||||
|
||||
.tabs button[aria-selected='true'] {
|
||||
color: var(--color-text);
|
||||
border-bottom-color: var(--color-primary);
|
||||
}
|
||||
|
||||
.bar {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
+53
-15
@@ -1,12 +1,46 @@
|
||||
import { it, expect, vi, beforeEach } from 'vitest';
|
||||
import { render, screen, fireEvent, waitFor } from '@testing-library/svelte';
|
||||
|
||||
const { session, ui } = vi.hoisted(() => ({
|
||||
const { session, ui, pageState } = vi.hoisted(() => ({
|
||||
session: { user: { id: '1', username: 'admin', role: 'admin' } },
|
||||
ui: { notify: vi.fn() }
|
||||
ui: { notify: vi.fn() },
|
||||
// Mock of SvelteKit's `$app/state` `page` — post-URL-routing
|
||||
// the admin page reads `page.params.tab` to derive which
|
||||
// section to render. Tests set the tab via `setTab(...)`
|
||||
// BEFORE `render(AdminPage)`; the derived picks it up on
|
||||
// initial mount. Previously the tab was chosen by clicking a
|
||||
// horizontal-tab button that no longer exists.
|
||||
pageState: {
|
||||
page: {
|
||||
url: new URL('http://localhost/admin'),
|
||||
params: {} as Record<string, string | undefined>,
|
||||
route: { id: '/admin/[[tab]]' },
|
||||
status: 200,
|
||||
error: null,
|
||||
data: {},
|
||||
form: null,
|
||||
state: {}
|
||||
}
|
||||
}
|
||||
}));
|
||||
vi.mock('$lib/stores/session.svelte', () => ({ session }));
|
||||
vi.mock('$lib/stores/ui.svelte', () => ({ ui }));
|
||||
vi.mock('$app/state', () => pageState);
|
||||
vi.mock('$app/navigation', () => ({ goto: vi.fn() }));
|
||||
vi.mock('$app/paths', () => ({ base: '', resolve: (r: string) => r }));
|
||||
|
||||
/**
|
||||
* Set the current tab BEFORE calling `render(AdminPage)`. The
|
||||
* admin page reads the URL-derived tab in a `$derived`, which
|
||||
* captures the value at first render — mutating this mock later
|
||||
* doesn't retrigger. Tests that exercise multiple tabs render
|
||||
* once per tab (each in a fresh `render` call — @testing-library
|
||||
* unmounts between tests via its `beforeEach` cleanup).
|
||||
*/
|
||||
function setTab(tab: string | undefined) {
|
||||
pageState.page.params = tab ? { tab } : {};
|
||||
pageState.page.url = new URL(`http://localhost/admin${tab ? '/' + tab : ''}`);
|
||||
}
|
||||
vi.mock('$lib/api/endpoints/admin', () => ({
|
||||
clearPluginLogs: vi.fn(),
|
||||
createExternalMount: vi.fn(),
|
||||
@@ -87,6 +121,10 @@ const mount = {
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
// Reset the tab mock so a test that sets `setTab('users')`
|
||||
// doesn't leak into the next test (default = /admin →
|
||||
// dashboard).
|
||||
setTab(undefined);
|
||||
m(admin.getDashboard).mockResolvedValue(dashboard);
|
||||
m(admin.listUsers).mockResolvedValue({ total: 1, users: [user] });
|
||||
m(admin.listPlugins).mockResolvedValue({ available: true, enabled: true, plugins: [] });
|
||||
@@ -136,8 +174,8 @@ it('toggles registration from the dashboard', async () => {
|
||||
|
||||
it('loads users when the users tab is opened and creates a user', async () => {
|
||||
m(admin.createUser).mockResolvedValue(undefined);
|
||||
setTab('users');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-users-tab'));
|
||||
await waitFor(() => expect(admin.listUsers).toHaveBeenCalled());
|
||||
await fireEvent.click(await screen.findByTestId('admin-users-create-btn'));
|
||||
await fireEvent.input(await screen.findByTestId('admin-create-user-username-input'), {
|
||||
@@ -151,33 +189,33 @@ it('loads users when the users tab is opened and creates a user', async () => {
|
||||
});
|
||||
|
||||
it('loads OIDC settings when the OIDC tab is opened', async () => {
|
||||
setTab('oidc');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-oidc-tab'));
|
||||
await waitFor(() => expect(admin.getOidcSettings).toHaveBeenCalled());
|
||||
});
|
||||
|
||||
it('loads storage + migration when the storage tab is opened', async () => {
|
||||
setTab('storage');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-storage-tab'));
|
||||
await waitFor(() => expect(admin.getStorageSettings).toHaveBeenCalled());
|
||||
await waitFor(() => expect(admin.getMigration).toHaveBeenCalled());
|
||||
});
|
||||
|
||||
it('loads SMTP info when the SMTP tab is opened', async () => {
|
||||
setTab('smtp');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-smtp-tab'));
|
||||
await waitFor(() => expect(admin.getSmtpInfo).toHaveBeenCalled());
|
||||
});
|
||||
|
||||
it('loads plugins when the plugins tab is opened', async () => {
|
||||
setTab('plugins');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-plugins-tab'));
|
||||
await waitFor(() => expect(admin.listPlugins).toHaveBeenCalled());
|
||||
});
|
||||
|
||||
it('loads external mounts when the mounts tab is opened and lists them', async () => {
|
||||
setTab('mounts');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-mounts-tab'));
|
||||
await waitFor(() => expect(admin.listExternalMounts).toHaveBeenCalled());
|
||||
// The configured mount is rendered in the table.
|
||||
expect(await screen.findByText('Media')).toBeTruthy();
|
||||
@@ -194,8 +232,8 @@ it('creates a mount from the mounts form', async () => {
|
||||
mount_path: 'Personal/Photos',
|
||||
config: { path: '/srv/photos', read_only: false }
|
||||
});
|
||||
setTab('mounts');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-mounts-tab'));
|
||||
await fireEvent.input(await screen.findByTestId('mount-name'), {
|
||||
target: { value: 'Photos' }
|
||||
});
|
||||
@@ -212,8 +250,8 @@ it('creates a mount from the mounts form', async () => {
|
||||
|
||||
it('deletes a mount through the confirm modal', async () => {
|
||||
m(admin.deleteExternalMount).mockResolvedValue(undefined);
|
||||
setTab('mounts');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-mounts-tab'));
|
||||
await fireEvent.click(await screen.findByTestId('mount-delete'));
|
||||
// deleteMount() gates on the styled confirm modal.
|
||||
await fireEvent.click(await screen.findByTestId('admin-confirm-ok-btn'));
|
||||
@@ -222,8 +260,8 @@ it('deletes a mount through the confirm modal', async () => {
|
||||
|
||||
it("toggles a user's role through the confirm modal", async () => {
|
||||
m(admin.setUserRole).mockResolvedValue(undefined);
|
||||
setTab('users');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-users-tab'));
|
||||
await fireEvent.click(await screen.findByTestId('admin-user-toggle-role-u1'));
|
||||
await fireEvent.click(await screen.findByTestId('admin-confirm-ok-btn'));
|
||||
await waitFor(() => expect(admin.setUserRole).toHaveBeenCalledWith('u1', 'admin'));
|
||||
@@ -231,8 +269,8 @@ it("toggles a user's role through the confirm modal", async () => {
|
||||
|
||||
it('deactivates a user through the confirm modal', async () => {
|
||||
m(admin.setUserActive).mockResolvedValue(undefined);
|
||||
setTab('users');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-users-tab'));
|
||||
await fireEvent.click(await screen.findByTestId('admin-user-toggle-active-u1'));
|
||||
await fireEvent.click(await screen.findByTestId('admin-confirm-ok-btn'));
|
||||
await waitFor(() => expect(admin.setUserActive).toHaveBeenCalledWith('u1', false));
|
||||
@@ -240,8 +278,8 @@ it('deactivates a user through the confirm modal', async () => {
|
||||
|
||||
it('saves OIDC settings from the OIDC form', async () => {
|
||||
m(admin.saveOidc).mockResolvedValue(undefined);
|
||||
setTab('oidc');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-oidc-tab'));
|
||||
await fireEvent.input(await screen.findByTestId('admin-oidc-issuer-input'), {
|
||||
target: { value: 'https://idp.test' }
|
||||
});
|
||||
@@ -251,8 +289,8 @@ it('saves OIDC settings from the OIDC form', async () => {
|
||||
|
||||
it('sends an SMTP test email', async () => {
|
||||
m(admin.sendSmtpTest).mockResolvedValue({ ok: true } as never);
|
||||
setTab('smtp');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-smtp-tab'));
|
||||
await fireEvent.input(await screen.findByTestId('admin-smtp-to-input'), {
|
||||
target: { value: 'to@x.test' }
|
||||
});
|
||||
@@ -263,8 +301,8 @@ it('sends an SMTP test email', async () => {
|
||||
it('saves storage settings and starts a migration', async () => {
|
||||
m(admin.saveStorage).mockResolvedValue(undefined);
|
||||
m(admin.migrationAction).mockResolvedValue(undefined);
|
||||
setTab('storage');
|
||||
render(AdminPage);
|
||||
await fireEvent.click(await screen.findByTestId('admin-storage-tab'));
|
||||
await fireEvent.submit(await screen.findByTestId('admin-storage-form'));
|
||||
await waitFor(() => expect(admin.saveStorage).toHaveBeenCalled());
|
||||
await fireEvent.click(await screen.findByTestId('admin-migration-start-btn'));
|
||||
@@ -806,7 +806,7 @@
|
||||
|
||||
<style>
|
||||
/* Filter cluster lives inside ResourceList's action-bar snippet now,
|
||||
but the actual DOM is scoped to THIS component's <style> block —
|
||||
but the actual DOM is scoped to THIS component's \3c style> block —
|
||||
Svelte's scoped selectors still apply because these are declared
|
||||
with the elements they style below.
|
||||
|
||||
|
||||
@@ -80,6 +80,8 @@
|
||||
"trash": "Trash",
|
||||
"groups": "Groups",
|
||||
"primary": "Primary",
|
||||
"admin_sections": "Admin sections",
|
||||
"back_to_app": "Back to OxiCloud",
|
||||
"profile": "Profile",
|
||||
"shared_with_me": "Shared with me",
|
||||
"toggle": "Toggle navigation menu"
|
||||
@@ -1191,7 +1193,74 @@
|
||||
"delete_user_title": "Delete user",
|
||||
"delete_user_warning": "You are about to permanently delete \"{{name}}\". This will remove the account, revoke every session, and reap the personal drive. This cannot be undone.",
|
||||
"delete_user_confirm_hint": "To confirm, type the account email below: {{email}}",
|
||||
"deleting": "Deleting…"
|
||||
"deleting": "Deleting…",
|
||||
"jobs": {
|
||||
"tab": "Jobs",
|
||||
"title": "Jobs",
|
||||
"hint": "Fires periodic + on-demand jobs. Consistency checks are safe to run at any time — they are read-only.",
|
||||
"run_all_consistency": "Run all consistency checks",
|
||||
"run_deep": "Run deep",
|
||||
"run_deep_hint": "Also runs slow variants (blob re-hash, bitrot detection).",
|
||||
"col_name": "Name",
|
||||
"col_cadence": "Cadence",
|
||||
"col_last_run": "Last run",
|
||||
"col_outcome": "Outcome",
|
||||
"col_state": "State",
|
||||
"col_actions": "Actions",
|
||||
"col_started_at": "Started",
|
||||
"col_status": "Status",
|
||||
"col_duration": "Duration",
|
||||
"col_scanned": "Scanned",
|
||||
"col_progress": "Progress",
|
||||
"col_findings": "Findings",
|
||||
"progress_exact_tooltip": "{{pct}} ({{scanned}} / {{total}})",
|
||||
"progress_approx_tooltip": "{{pct}} ({{scanned}} / {{total}} — approximate, backend proxy)",
|
||||
"progress_scanned_only": "{{n}} scanned",
|
||||
"progress_scanned_only_tooltip": "No total available for this run (pre-progress-bar deploy or the tenant does not report a countable subject).",
|
||||
"findings_present_tooltip": "Expand this run to see per-finding detail.",
|
||||
"col_error": "Error",
|
||||
"col_kind": "Kind",
|
||||
"col_severity": "Severity",
|
||||
"col_resource": "Resource",
|
||||
"col_detail": "Detail",
|
||||
"run": "Run",
|
||||
"cancel": "Cancel",
|
||||
"refresh": "Refresh",
|
||||
"runs_title": "Recent runs",
|
||||
"run_json": "Run summary (JSON)",
|
||||
"findings_title": "Findings",
|
||||
"no_runs": "No runs yet.",
|
||||
"no_findings": "No findings — clean run.",
|
||||
"none_registered": "No jobs registered.",
|
||||
"on_demand": "on-demand",
|
||||
"every_h": "every {{n}} h",
|
||||
"every_min": "every {{n}} min",
|
||||
"every_sec": "every {{n}} s",
|
||||
"outcome_ok": "ok",
|
||||
"outcome_err": "err",
|
||||
"outcome_issues": "issues",
|
||||
"outcome_notices": "notices",
|
||||
"n_findings": "{{n}} findings",
|
||||
"n_notices": "{{n}} notices",
|
||||
"notices_present_tooltip": "Informational findings — no action required. Expand for detail.",
|
||||
"purge": "Purge old runs",
|
||||
"purge_hint": "Delete completed and failed run history older than the chosen retention window. Findings drop with their parent runs. Non-terminal runs are always preserved.",
|
||||
"purge_title": "Purge old job history",
|
||||
"purge_body": "Delete completed and failed run history older than the chosen number of days. Findings drop with their parent runs. Non-terminal runs (running, paused, cancel-requested) are always preserved.",
|
||||
"purge_days_label": "Retention (days)",
|
||||
"purge_confirm": "Purge",
|
||||
"purge_done": "{{n}} old run(s) purged (retention {{days}} days)",
|
||||
"state_running": "running",
|
||||
"never": "never",
|
||||
"just_now": "just now",
|
||||
"n_min_ago": "{{n}} min ago",
|
||||
"n_h_ago": "{{n}} h ago",
|
||||
"n_d_ago": "{{n}} d ago",
|
||||
"triggered_ok": "{{name}} triggered successfully",
|
||||
"triggered_err": "{{name}} failed: {{message}}",
|
||||
"cancel_requested": "Cancel requested — {{name}} will pause at the next safe boundary",
|
||||
"cancel_noop": "Nothing to cancel — {{name}} is not running"
|
||||
}
|
||||
},
|
||||
"profile": {
|
||||
"page_title": "Profile",
|
||||
|
||||
@@ -71,14 +71,14 @@ coverage-integration filter='mount':
|
||||
# of this file would otherwise leak in) cannot point the tests at the
|
||||
# real dev DB. The test pool helpers also refuse non-`oxicloud_test`
|
||||
# URLs as defence in depth.
|
||||
test-integration:
|
||||
test-integration filter='':
|
||||
bash tests/common/spawn-db.sh
|
||||
PGHOST=localhost PGPORT=5433 PGUSER=oxicloud_test PGPASSWORD=oxicloud_test \
|
||||
PGDATABASE=oxicloud_test \
|
||||
bash tests/common/init-test-schema.sh
|
||||
DATABASE_URL='postgres://oxicloud_test:oxicloud_test@localhost:5433/oxicloud_test' \
|
||||
RUSTFLAGS='--cfg integration_tests' \
|
||||
cargo test --workspace --tests
|
||||
cargo test --workspace --tests {{filter}}
|
||||
bash tests/common/stop-db.sh
|
||||
|
||||
test-one name:
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
-- ============================================================================
|
||||
-- Slice 1 of Part 2 (Recoverable-Run Engine) — see
|
||||
-- docs/plan/job-registry.md#part-2--recoverable-run-engine.
|
||||
--
|
||||
-- Introduces the `jobs.*` schema housing state for long-running,
|
||||
-- restart-tolerant jobs (storage migration, reextract-*, consistency
|
||||
-- checks). Deliberately distinct from `auth.*` / `storage.*` / `admin.*`
|
||||
-- because this is JOB-runtime state, not domain data.
|
||||
--
|
||||
-- This migration lands the ENGINE table only. Job-specific per-record
|
||||
-- artifacts (the `jobs.run_findings` generic table used by consistency
|
||||
-- checks + migration failure logs + reextract failures) land alongside
|
||||
-- Slice 2 or its first tenant PR — deferred here to keep the review
|
||||
-- surface tight.
|
||||
-- ============================================================================
|
||||
|
||||
CREATE SCHEMA IF NOT EXISTS jobs;
|
||||
|
||||
-- ─── recoverable_runs ────────────────────────────────────────────────────────
|
||||
-- One row per RUN of a RecoverableJobHandler tenant. Non-terminal
|
||||
-- rows carry the live cursor + stats; terminal rows are the audit
|
||||
-- trail (last-run visibility for `GET /api/admin/jobs`, retention
|
||||
-- pruning deferred).
|
||||
--
|
||||
-- `status` values (TEXT, enforced by the partial unique index below
|
||||
-- plus per-tenant discipline):
|
||||
-- • 'Running' — actively executing.
|
||||
-- • 'Paused' — cooperatively yielded (cancel poll or
|
||||
-- graceful shutdown). Resumable from `cursor`.
|
||||
-- • 'CancelRequested' — cancel signalled; handler is winding down.
|
||||
-- Still non-terminal — a re-trigger must NOT
|
||||
-- spawn a parallel run.
|
||||
-- • 'Completed' — walked the whole space. Terminal.
|
||||
-- • 'Failed' — irrecoverable error. Terminal.
|
||||
-- `error_message` is populated.
|
||||
--
|
||||
-- Column notes:
|
||||
-- • `started_at` is FIXED at run start — long-running consistency
|
||||
-- scans use it as their grace-window reference (see trap #1 in
|
||||
-- docs/plan/consistency-check.md).
|
||||
-- • `last_progress_at` bumps on every checkpoint. Doubles as heartbeat
|
||||
-- for the boot-time crash-recovery sweep (Running rows with stale
|
||||
-- heartbeats get flipped to Paused on server restart).
|
||||
-- • `cursor` is opaque bytes per-job (BLAKE3 hash for blob scans,
|
||||
-- UUID for file scans, ltree path for folder-tree scans). NULL on
|
||||
-- a fresh run's first checkpoint.
|
||||
-- • `stats` accumulates per-run counters (scanned_count,
|
||||
-- migrated_blobs, findings_this_run, …). JSONB shape is per-job;
|
||||
-- the reserved top-level `count` mirrors JobOutcome::Ok.count.
|
||||
-- • `params` carries per-run params captured at start (grace_window_secs,
|
||||
-- source_backend, …). Distinct from stats — params are set once,
|
||||
-- stats accumulate.
|
||||
|
||||
CREATE TABLE jobs.recoverable_runs (
|
||||
id UUID PRIMARY KEY,
|
||||
job_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
started_at TIMESTAMPTZ NOT NULL,
|
||||
last_progress_at TIMESTAMPTZ NOT NULL,
|
||||
completed_at TIMESTAMPTZ,
|
||||
cursor BYTEA,
|
||||
stats JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
params JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
error_message TEXT
|
||||
);
|
||||
|
||||
-- Exclusivity — the load-bearing invariant.
|
||||
--
|
||||
-- "At most one non-terminal run per job_name." Enforced at the DB
|
||||
-- layer so concurrent triggers or scheduler-vs-operator races cannot
|
||||
-- create a parallel run. The partial index makes duplicate INSERTs
|
||||
-- fail with a unique-violation; the trigger-endpoint handler catches
|
||||
-- that and returns the surviving row instead.
|
||||
--
|
||||
-- CancelRequested is INCLUDED — a second trigger during cancel must
|
||||
-- not spawn a parallel run.
|
||||
CREATE UNIQUE INDEX one_active_run_per_job
|
||||
ON jobs.recoverable_runs (job_name)
|
||||
WHERE status IN ('Running', 'Paused', 'CancelRequested');
|
||||
|
||||
-- Boot recovery sweep index. On restart, `UPDATE ... SET status='Paused'
|
||||
-- WHERE status='Running' OR status='CancelRequested'` finds all abandoned
|
||||
-- rows. Partial index keeps it a fast index-only scan even when the table
|
||||
-- accumulates terminal (Completed/Failed) rows over time.
|
||||
CREATE INDEX ON jobs.recoverable_runs (last_progress_at)
|
||||
WHERE status = 'Running';
|
||||
|
||||
-- "Latest run per job" query — powers `GET /api/admin/jobs` when
|
||||
-- recoverable jobs appear in the listing (`SELECT DISTINCT ON (job_name)
|
||||
-- ... ORDER BY job_name, started_at DESC`).
|
||||
CREATE INDEX ON jobs.recoverable_runs (job_name, started_at DESC);
|
||||
@@ -0,0 +1,62 @@
|
||||
-- ============================================================================
|
||||
-- Slice 7 of Part 2 — persistent finding storage.
|
||||
--
|
||||
-- Findings from consistency jobs (and eventually storage_migration failure
|
||||
-- rows, reextract failures) currently flow into `tracing::warn!` targeted
|
||||
-- at `oxicloud::consistency`. That works for live tailing but rotates
|
||||
-- away — an operator opening the admin UI a day after a run has nothing
|
||||
-- to drill into. This table makes findings first-class + queryable.
|
||||
--
|
||||
-- `run_findings` is INTENTIONALLY generic. The consistency-check plan and
|
||||
-- the plan doc's tenant table both list `kind` + `severity` + `resource_id`
|
||||
-- + `detail` as the union of what every current tenant needs. Adding a
|
||||
-- tenant with a novel per-finding field means widening `detail`
|
||||
-- (JSONB, per-tenant shape), not adding a column.
|
||||
--
|
||||
-- Cascade rule: findings live and die with their parent run. Deleting a
|
||||
-- terminal run row (retention pruning, planned) also drops its findings.
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE jobs.run_findings (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
run_id UUID NOT NULL
|
||||
REFERENCES jobs.recoverable_runs(id)
|
||||
ON DELETE CASCADE,
|
||||
-- Machine-readable kind (e.g. 'stale_used_bytes', 'missing_blob').
|
||||
-- Stable across releases per the audit-log convention — see
|
||||
-- feedback memory `enum_over_string_literals_in_logs`. New failure
|
||||
-- mode = new value, never repurpose an existing one.
|
||||
kind TEXT NOT NULL,
|
||||
-- Severity spectrum:
|
||||
-- 'data_loss' — bytes / rows unreachable or gone.
|
||||
-- 'inconsistent' — counters / materialised values wrong,
|
||||
-- content intact.
|
||||
-- 'anomaly' — surprising state worth surfacing, no known impact.
|
||||
-- TEXT (not ENUM) so tenants can grow the vocabulary without a schema
|
||||
-- migration; the app-layer types.rs is where the canonical set lives.
|
||||
severity TEXT NOT NULL,
|
||||
-- Nullable — some findings pertain to the run as a whole
|
||||
-- (e.g. "backend enumeration truncated after 1M keys") rather than
|
||||
-- one specific resource.
|
||||
resource_id UUID,
|
||||
-- Per-tenant per-finding structured detail (cached/actual/delta,
|
||||
-- blob_hash, expected/stored path, ...). Consumers key off `kind`
|
||||
-- to know the shape.
|
||||
detail JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Time-ordered listing per run: `GET /api/admin/jobs/{name}/runs/{id}/findings`
|
||||
-- pages by (run_id, created_at) — index-only scan.
|
||||
CREATE INDEX ON jobs.run_findings (run_id, created_at);
|
||||
|
||||
-- Aggregation queries — "how many `missing_blob` findings across all
|
||||
-- runs of `files_consistency`" or "how many `data_loss`-severity
|
||||
-- findings today". Both need `(kind)` and `(severity)` predicates; a
|
||||
-- composite works for either since the leading column is selective.
|
||||
CREATE INDEX ON jobs.run_findings (kind, severity, created_at);
|
||||
|
||||
COMMENT ON TABLE jobs.run_findings IS
|
||||
'Structured per-finding records emitted by recoverable jobs. Replaces '
|
||||
'the transitional `tracing::warn!(event=consistency_finding)` calls '
|
||||
'in the consistency tenants — see docs/plan/job-registry.md Part 2.';
|
||||
@@ -10,6 +10,7 @@
|
||||
//! and PostgreSQL index logic in `DedupService` itself.
|
||||
|
||||
use bytes::Bytes;
|
||||
use chrono::{DateTime, Utc};
|
||||
use futures::Stream;
|
||||
use serde::Serialize;
|
||||
use std::future::Future;
|
||||
@@ -18,6 +19,53 @@ use std::pin::Pin;
|
||||
|
||||
use crate::domain::errors::DomainError;
|
||||
|
||||
/// One row returned by [`BlobStorageBackend::list_blob_hashes`] — the
|
||||
/// hash of a blob physically present on the backend, plus its
|
||||
/// last-modified timestamp when the backend can supply one. `mtime`
|
||||
/// is used by `backend_consistency` to skip freshly-created files
|
||||
/// still within the write grace window (avoids false-positive
|
||||
/// orphans during the durability-before-visibility window that
|
||||
/// `dedup_service` opens).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackendBlobEntry {
|
||||
pub hash: String,
|
||||
/// `None` when the backend doesn't track mtime — the consistency
|
||||
/// scan then falls back to treating the entry as "old enough" and
|
||||
/// will emit an orphan finding without a grace check.
|
||||
pub mtime: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// A file present in the blob-storage namespace but NOT matching the
|
||||
/// canonical `<64-hex>.blob` shape. Sidecars (`.blob.orig`,
|
||||
/// `.blob.lost`, `.blob.tmp`), wrong extensions, non-hex names —
|
||||
/// anything the enumeration filter skips for the blob list. Surfaced
|
||||
/// so `backend_consistency` can emit them as `anomaly` notices
|
||||
/// (informational only — they don't hurt anything but the operator
|
||||
/// should know they're there).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackendUnknownEntry {
|
||||
/// Backend-relative path (`04/04f48c...blob.orig` on local FS or
|
||||
/// as an S3 key). Included in the finding detail so operators
|
||||
/// can locate it.
|
||||
pub path: String,
|
||||
pub mtime: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Return type of [`BlobStorageBackend::list_blob_hashes`] — one
|
||||
/// batch of the enumeration. Struct (not tuple) so adding future
|
||||
/// per-batch metadata (e.g. `truncated: bool`) doesn't break every
|
||||
/// backend impl. `next_cursor = None` signals end of enumeration.
|
||||
///
|
||||
/// Backends that don't track sidecar/unknown files leave `unknowns`
|
||||
/// empty; the tenant just doesn't emit any `unknown_backend_file`
|
||||
/// notices from that batch.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlobListPage {
|
||||
pub blobs: Vec<BackendBlobEntry>,
|
||||
pub unknowns: Vec<BackendUnknownEntry>,
|
||||
pub next_cursor: Option<String>,
|
||||
}
|
||||
|
||||
/// Boxed future alias used by [`BlobStorageBackend`] to keep the trait dyn-compatible.
|
||||
type BoxFut<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
|
||||
@@ -145,4 +193,40 @@ pub trait BlobStorageBackend: Send + Sync + 'static {
|
||||
fn read_prefetch(&self) -> usize {
|
||||
1
|
||||
}
|
||||
|
||||
/// Enumerate blob entries physically present on this backend, in
|
||||
/// implementation-defined order — cursor-based paging.
|
||||
///
|
||||
/// * `cursor` — opaque continuation token from a prior call, or
|
||||
/// `None` to start from the beginning. Format is per-backend
|
||||
/// (local = last path visited; S3 = continuation token; Azure
|
||||
/// = list marker); callers treat it as opaque.
|
||||
/// * `limit` — soft cap on batch size; backends may return
|
||||
/// fewer (e.g. end of a shard directory).
|
||||
///
|
||||
/// Returns `(entries, next_cursor)`. `next_cursor = None` means
|
||||
/// enumeration is complete. Each `BackendBlobEntry` carries the
|
||||
/// hash + optional mtime for grace-window filtering.
|
||||
///
|
||||
/// The trait default returns
|
||||
/// [`DomainError::NotSupported`](DomainError::not_supported)
|
||||
/// — future backends that genuinely can't enumerate (some
|
||||
/// write-only queue, some read-only mirror) can inherit it. All
|
||||
/// currently-shipped backends (local, S3, Azure) override.
|
||||
///
|
||||
/// Filtering out non-blob artifacts (temp files, `.corrupt` /
|
||||
/// `.lost` sidecars, encryption metadata) is the backend's
|
||||
/// responsibility — the tenant walks whatever this returns.
|
||||
fn list_blob_hashes(
|
||||
&self,
|
||||
_cursor: Option<String>,
|
||||
_limit: usize,
|
||||
) -> BoxFut<'_, Result<BlobListPage, DomainError>> {
|
||||
Box::pin(async {
|
||||
Err(DomainError::operation_not_supported(
|
||||
"list_blob_hashes",
|
||||
"this backend does not implement enumeration",
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -578,9 +578,25 @@ impl StorageUsageService {
|
||||
|
||||
pub const STORAGE_RECONCILE_JOB_NAME: &str = "storage_reconcile";
|
||||
|
||||
use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRunArgs};
|
||||
use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs};
|
||||
use async_trait::async_trait;
|
||||
|
||||
impl StorageUsageService {
|
||||
/// Register self with the periodic-job scheduler and return the
|
||||
/// same `Arc<Self>` for DI-style chaining. Scheduled tenant with
|
||||
/// interval = `max(30s, interval_secs)`. See
|
||||
/// `docs/plan/job-registry.md` Part 1.
|
||||
pub async fn register(
|
||||
self: Arc<Self>,
|
||||
registry: &JobRegistry,
|
||||
interval_secs: u64,
|
||||
) -> Arc<Self> {
|
||||
let interval = Self::reconciliation_interval(interval_secs);
|
||||
registry.register(self.clone(), Some(interval), None).await;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for StorageUsageService {
|
||||
fn name(&self) -> &str {
|
||||
|
||||
+196
-77
@@ -322,6 +322,13 @@ impl AppServiceFactory {
|
||||
let blob_lifecycle =
|
||||
Arc::new(BlobLifecycleService::new().with_hook(thumbnail_service.clone()));
|
||||
|
||||
// Hold a clone of the fully-decorated blob backend for use by
|
||||
// `blobs_consistency` (physical-existence + bit-rot probes)
|
||||
// further down the DI chain. Must be captured BEFORE the
|
||||
// `dedup_service` construction below because that call moves
|
||||
// `blob_backend` into DedupService.
|
||||
let blob_backend_for_consistency = blob_backend.clone();
|
||||
|
||||
// Deduplication service — PRIMARY blob storage engine (PostgreSQL-backed index)
|
||||
let dedup_service = Arc::new(
|
||||
crate::infrastructure::services::dedup_service::DedupService::new(
|
||||
@@ -436,6 +443,15 @@ impl AppServiceFactory {
|
||||
// have landed.
|
||||
let job_registry = Arc::new(JobRegistry::new());
|
||||
|
||||
// Recoverable-run engine's PG provider (`jobs.recoverable_runs`).
|
||||
// Runs on the maintenance pool so a long-running scan's cursor
|
||||
// updates never contend with the request-path pool. Boot-time
|
||||
// crash-recovery sweep fires in `build_app_state` after the
|
||||
// provider is placed on `AppState`.
|
||||
let job_store_provider = Arc::new(
|
||||
crate::infrastructure::scheduler::PgJobStoreProvider::new(maintenance_pool.clone()),
|
||||
);
|
||||
|
||||
Ok(CoreServices {
|
||||
path_service,
|
||||
file_content_cache,
|
||||
@@ -449,6 +465,8 @@ impl AppServiceFactory {
|
||||
zip_service: None, // Placeholder - replaced after app services init
|
||||
config: self.config.clone(),
|
||||
job_registry,
|
||||
job_store_provider,
|
||||
blob_backend: blob_backend_for_consistency,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -878,27 +896,18 @@ impl AppServiceFactory {
|
||||
// (`docs/plan/job-registry.md` Part 1) instead of spawning its own
|
||||
// tokio interval loop. `SchedulerEngine::start` fires the actual
|
||||
// supervisor task at the end of `build_app_state`.
|
||||
let cleanup_service = Arc::new(TrashCleanupService::new(
|
||||
// Self-registering constructor chain — TrashCleanupService owns
|
||||
// its interval + timeout shape; DI only supplies deps.
|
||||
// `.register(®)` fires the uniform `job.registered` log line
|
||||
// and panics on wiring error (duplicate name = boot must fail
|
||||
// loud). See `docs/plan/job-registry.md` Part 1.
|
||||
let _ = Arc::new(TrashCleanupService::new(
|
||||
trash_repo.clone(),
|
||||
core.dedup_service.clone(),
|
||||
24, // Run cleanup every 24 hours
|
||||
));
|
||||
let interval = cleanup_service.interval();
|
||||
if let Err(e) = core
|
||||
.job_registry
|
||||
.register(cleanup_service.clone(), Some(interval), None)
|
||||
.await
|
||||
{
|
||||
// Duplicate registration is the only failure mode today and
|
||||
// shouldn't happen in the normal DI flow. Log + continue so
|
||||
// trash service still lands even if scheduling didn't.
|
||||
tracing::error!("Failed to register trash_cleanup job with scheduler: {e}");
|
||||
} else {
|
||||
tracing::info!(
|
||||
"Trash cleanup registered with scheduler (interval {} h)",
|
||||
interval.as_secs() / 3600
|
||||
);
|
||||
}
|
||||
))
|
||||
.register(&core.job_registry)
|
||||
.await;
|
||||
|
||||
Some(service as Arc<TrashService>)
|
||||
}
|
||||
@@ -1125,7 +1134,12 @@ impl AppServiceFactory {
|
||||
// and invalidation would be a no-op observed by nobody —
|
||||
// this is the trap that regressed the used_bytes freshness
|
||||
// after perf commit `12dc648c`.
|
||||
let service = Arc::new(
|
||||
// Keep cached storage usage fresh off the request path: GET
|
||||
// /api/auth/me no longer recomputes the O(N) SUM per call; a
|
||||
// periodic sweep does it instead (on the maintenance pool).
|
||||
// Self-registering constructor chain — StorageUsageService owns
|
||||
// its interval-clamping via `Self::reconciliation_interval`.
|
||||
Arc::new(
|
||||
crate::application::services::storage_usage_service::StorageUsageService::new(
|
||||
maintenance_pool.clone(),
|
||||
user_repository,
|
||||
@@ -1134,28 +1148,9 @@ impl AppServiceFactory {
|
||||
drive_repo
|
||||
as Arc<dyn crate::domain::repositories::drive_repository::DriveRepository>,
|
||||
),
|
||||
);
|
||||
// Keep cached storage usage fresh off the request path: GET
|
||||
// /api/auth/me no longer recomputes the O(N) SUM per call; a
|
||||
// periodic sweep does it instead (on the maintenance pool).
|
||||
// Registered with the periodic-job scheduler
|
||||
// (`docs/plan/job-registry.md` Part 1); the retired
|
||||
// `start_reconciliation_job` used to spawn its own interval loop.
|
||||
let interval =
|
||||
StorageUsageService::reconciliation_interval(self.config.storage.usage_reconcile_secs);
|
||||
if let Err(e) = core
|
||||
.job_registry
|
||||
.register(service.clone(), Some(interval), None)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Failed to register storage_reconcile job: {e}");
|
||||
} else {
|
||||
tracing::info!(
|
||||
"Storage-usage reconciliation registered with scheduler (interval {}s)",
|
||||
interval.as_secs()
|
||||
);
|
||||
}
|
||||
service
|
||||
)
|
||||
.register(&core.job_registry, self.config.storage.usage_reconcile_secs)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Starts the tree-ETag flush job (requires database).
|
||||
@@ -1279,22 +1274,112 @@ impl AppServiceFactory {
|
||||
// Register on-demand-only jobs whose owning service lives on
|
||||
// CoreServices. Dedup GC has NO periodic tick — trash cleanup's
|
||||
// sweep already runs GC as its tail step, so a periodic dedup
|
||||
// schedule would double the work. Registering with `interval =
|
||||
// None` keeps it admin-triggerable through the uniform scheduler
|
||||
// surface (`POST /api/admin/jobs/dedup_gc/trigger`).
|
||||
if let Err(e) = core
|
||||
.job_registry
|
||||
.register(
|
||||
core.dedup_service.clone() as Arc<dyn crate::infrastructure::scheduler::JobHandler>,
|
||||
None, // on-demand only
|
||||
None, // no timeout
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Failed to register dedup_gc job with scheduler: {e}");
|
||||
} else {
|
||||
tracing::info!("Dedup GC registered with scheduler (on-demand only)");
|
||||
}
|
||||
// schedule would double the work. The `register()` method
|
||||
// encapsulates the on-demand shape.
|
||||
let _ = core
|
||||
.dedup_service
|
||||
.clone()
|
||||
.register(&core.job_registry)
|
||||
.await;
|
||||
|
||||
// First recoverable-run tenant (`docs/plan/job-registry.md`
|
||||
// Part 2). Iterates `storage.drives` and reports each drive
|
||||
// whose cached `used_bytes` differs from `SUM(files.size)`.
|
||||
// On-demand only — read-only diagnostic, not periodic.
|
||||
// Runs on the maintenance pool alongside the other sweeps.
|
||||
let job_store_provider_dyn: Arc<dyn crate::infrastructure::scheduler::JobStoreProvider> =
|
||||
core.job_store_provider.clone();
|
||||
let _ = Arc::new(
|
||||
crate::infrastructure::services::drives_consistency_service::DrivesConsistencyCheck::new(
|
||||
maintenance_pool.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&core.job_registry, &job_store_provider_dyn)
|
||||
.await;
|
||||
|
||||
// Second recoverable-run tenant. Iterates `storage.folders`
|
||||
// and reports each row whose materialised path/lpath or
|
||||
// parent-trashed state has drifted from the parent-chain
|
||||
// reconstruction — same subject-iteration pattern as drives.
|
||||
// On-demand only; findings surface via the
|
||||
// `oxicloud::consistency` tracing target until the
|
||||
// `jobs.run_findings` table lands.
|
||||
let _ = Arc::new(
|
||||
crate::infrastructure::services::folders_consistency_service::FoldersConsistencyCheck::new(
|
||||
maintenance_pool.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&core.job_registry, &job_store_provider_dyn)
|
||||
.await;
|
||||
|
||||
// Third recoverable-run tenant. Iterates `storage.files`
|
||||
// and reports parent-folder-trashed cascade misses,
|
||||
// `missing_blob` (data-loss indicator — file references
|
||||
// absent blob row), and `blob_size_mismatch` (denormalised
|
||||
// size drift). One SQL round-trip loads folder + blob via
|
||||
// two LEFT JOINs; per-row branches key off the join
|
||||
// results.
|
||||
let _ = Arc::new(
|
||||
crate::infrastructure::services::files_consistency_service::FilesConsistencyCheck::new(
|
||||
maintenance_pool.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&core.job_registry, &job_store_provider_dyn)
|
||||
.await;
|
||||
|
||||
// Fourth recoverable-run tenant. Iterates `storage.blobs`
|
||||
// and verifies each row against the physical backend AND
|
||||
// against the reference-counting invariants that `dedup_gc`
|
||||
// relies on. Three per-row checks (subject-iteration in
|
||||
// action): `blob_missing_from_backend` (data_loss, bytes
|
||||
// gone from disk), `refcount_mismatch` (inconsistent,
|
||||
// dedup counter drift), and `blob_corrupted` (data_loss,
|
||||
// deep mode only — bit-rot). Complements
|
||||
// `files_consistency` without doubling work: probing
|
||||
// per-unique-blob preserves dedup savings vs probing
|
||||
// per-file-chunk. See memory
|
||||
// `project_cdc_dual_storage_registries` for the rationale.
|
||||
let _ = Arc::new(
|
||||
crate::infrastructure::services::blobs_consistency_service::BlobsConsistencyCheck::new(
|
||||
maintenance_pool.clone(),
|
||||
core.blob_backend.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&core.job_registry, &job_store_provider_dyn)
|
||||
.await;
|
||||
|
||||
// Fifth recoverable-run tenant. Iterates the storage
|
||||
// backend (via `BlobStorageBackend::list_blob_hashes`) and
|
||||
// reports every physical blob that has no matching row in
|
||||
// `storage.blobs`. Closes the reference graph together with
|
||||
// `blobs_consistency`: this tenant walks backend→DB, that
|
||||
// one walks DB→backend. Enumeration is backend-specific but
|
||||
// the tenant is fully backend-agnostic — each backend owns
|
||||
// its own layout knowledge (local walks `.blobs/`, S3 uses
|
||||
// ListObjectsV2, migration wrapper refuses mid-migration).
|
||||
let _ = Arc::new(
|
||||
crate::infrastructure::services::backend_consistency_service::BackendConsistencyCheck::new(
|
||||
maintenance_pool.clone(),
|
||||
core.blob_backend.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&core.job_registry, &job_store_provider_dyn)
|
||||
.await;
|
||||
|
||||
// "Run all consistency checks" coordinator. Plain JobHandler
|
||||
// (not RecoverableJobHandler) — it dispatches, doesn't scan.
|
||||
// MUST register AFTER every `*_consistency` tenant so the
|
||||
// snapshot ordering in `GET /api/admin/jobs` shows children
|
||||
// then wrapper; snapshot filtering happens at run time so
|
||||
// late registration is fine. Weak<JobRegistry> internally
|
||||
// breaks the Arc cycle.
|
||||
let _ = Arc::new(
|
||||
crate::infrastructure::services::consistency_batch_service::ConsistencyBatch::new(
|
||||
&core.job_registry,
|
||||
),
|
||||
)
|
||||
.register_job(&core.job_registry)
|
||||
.await;
|
||||
|
||||
// 2. Repository services (requires PgPool for all metadata)
|
||||
let repos = self.create_repository_services(&core, &pool);
|
||||
@@ -1484,32 +1569,18 @@ impl AppServiceFactory {
|
||||
self.start_content_index_job(&maintenance_pool, &core, content_index);
|
||||
|
||||
grant_cleanup_service = if core.config.features.grant_cleanup.enabled {
|
||||
// Self-registering constructor chain. Grant-cleanup owns
|
||||
// its interval + on `?force=true` handling; DI only decides
|
||||
// whether to instantiate at all (feature-gated).
|
||||
let svc = Arc::new(
|
||||
crate::infrastructure::services::grant_cleanup_service::GrantCleanupService::new(
|
||||
authorization.clone(),
|
||||
core.config.features.grant_cleanup.grace_days,
|
||||
core.config.features.grant_cleanup.interval_hours,
|
||||
),
|
||||
);
|
||||
// Registered with the periodic-job scheduler
|
||||
// (`docs/plan/job-registry.md` Part 1); the retired
|
||||
// `start_cleanup_job` used to spawn its own interval loop.
|
||||
// Admin `?force=true` trigger still calls `svc.purge(Some(0))`
|
||||
// directly — grace override doesn't fit the JobHandler shape.
|
||||
let interval = svc.interval();
|
||||
if let Err(e) = core
|
||||
.job_registry
|
||||
.register(svc.clone(), Some(interval), None)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Failed to register grant_cleanup job: {e}");
|
||||
} else {
|
||||
tracing::info!(
|
||||
"Grant cleanup registered with scheduler (every {}h, grace = {}d)",
|
||||
interval.as_secs() / 3600,
|
||||
core.config.features.grant_cleanup.grace_days,
|
||||
);
|
||||
}
|
||||
)
|
||||
.register(&core.job_registry)
|
||||
.await;
|
||||
Some(svc)
|
||||
} else {
|
||||
tracing::info!(
|
||||
@@ -2172,6 +2243,41 @@ impl AppServiceFactory {
|
||||
}
|
||||
}
|
||||
|
||||
// Recoverable-run engine crash recovery. Any row still marked
|
||||
// Running or CancelRequested when the previous process died gets
|
||||
// flipped to Paused with `error_message = 'server restart mid-run'`.
|
||||
// We do NOT auto-resume — operators explicitly re-trigger via
|
||||
// `POST /api/admin/jobs/{name}/trigger`, which resumes from the
|
||||
// persisted cursor. Runs BEFORE the scheduler starts so a
|
||||
// periodic-triggered recoverable job's first tick sees a clean
|
||||
// slate. See `docs/plan/job-registry.md` Part 2.
|
||||
use crate::infrastructure::scheduler::JobStoreProvider as _;
|
||||
match app_state
|
||||
.core
|
||||
.job_store_provider
|
||||
.boot_recovery_sweep()
|
||||
.await
|
||||
{
|
||||
Ok(0) => tracing::debug!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "recoverable.boot_recovery",
|
||||
flipped = 0,
|
||||
"no orphaned recoverable runs found at boot"
|
||||
),
|
||||
Ok(n) => tracing::warn!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "recoverable.boot_recovery",
|
||||
flipped = n,
|
||||
"flipped {n} orphaned recoverable run(s) Running/CancelRequested → Paused (previous process died mid-run)"
|
||||
),
|
||||
Err(e) => tracing::error!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "recoverable.boot_recovery.failed",
|
||||
error = %e,
|
||||
"boot recovery sweep failed — orphaned runs may remain in Running state"
|
||||
),
|
||||
}
|
||||
|
||||
// Start the periodic-job scheduler AFTER every native service has
|
||||
// finished registering its jobs on `core.job_registry`. Starting
|
||||
// it earlier would race the first tick against late registrations.
|
||||
@@ -2212,6 +2318,19 @@ pub struct CoreServices {
|
||||
/// themselves here during their creation; `SchedulerEngine::start`
|
||||
/// spins up the supervisor loop at the end of `build_app_state`.
|
||||
pub job_registry: Arc<JobRegistry>,
|
||||
/// PG-backed provider for `jobs.recoverable_runs`. Recoverable
|
||||
/// tenants (storage migration, reextract, consistency checks —
|
||||
/// Part 2 of `docs/plan/job-registry.md`) plug into this via
|
||||
/// `svc.register_recoverable_job(®istry, &job_store_provider).await`.
|
||||
/// Boot-time crash-recovery sweep is run in `build_app_state` right
|
||||
/// after this provider is created.
|
||||
pub job_store_provider: Arc<crate::infrastructure::scheduler::PgJobStoreProvider>,
|
||||
/// Fully-decorated blob backend (retry → encryption → cache
|
||||
/// stack applied). Exposed here so tenants outside
|
||||
/// `create_core_services` — notably `blobs_consistency` in
|
||||
/// `build_app_state` — can probe `blob_exists()` / re-hash bytes
|
||||
/// through the same stack DedupService uses.
|
||||
pub blob_backend: Arc<dyn BlobStorageBackend>,
|
||||
}
|
||||
|
||||
/// Container for repository services
|
||||
|
||||
@@ -264,6 +264,10 @@ fn translate_join(
|
||||
/// Distinct Ok/Err branches so the tracing macros pick up the fields at
|
||||
/// compile time — `tracing` doesn't expand conditional field lists.
|
||||
fn log_outcome(name: &str, outcome: &JobOutcome, cause: Option<ErrCause>, elapsed_ms: u128) {
|
||||
// Also render elapsed inline in the human-readable message so
|
||||
// `tail -f` operators see the duration without waiting on a
|
||||
// structured log renderer to project the `elapsed_ms` field.
|
||||
let elapsed = format_elapsed(elapsed_ms);
|
||||
match outcome {
|
||||
JobOutcome::Ok { count, extra } => {
|
||||
tracing::info!(
|
||||
@@ -274,8 +278,10 @@ fn log_outcome(name: &str, outcome: &JobOutcome, cause: Option<ErrCause>, elapse
|
||||
count = *count,
|
||||
elapsed_ms = elapsed_ms,
|
||||
extra = %extra,
|
||||
"job {} ran",
|
||||
"job {} ran in {} — count={}",
|
||||
name,
|
||||
elapsed,
|
||||
count,
|
||||
);
|
||||
}
|
||||
JobOutcome::Err { message: msg } => {
|
||||
@@ -287,13 +293,31 @@ fn log_outcome(name: &str, outcome: &JobOutcome, cause: Option<ErrCause>, elapse
|
||||
cause = %cause.unwrap_or(ErrCause::Handler),
|
||||
elapsed_ms = elapsed_ms,
|
||||
error = %msg,
|
||||
"job {} failed",
|
||||
"job {} failed after {} — {}",
|
||||
name,
|
||||
elapsed,
|
||||
msg,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Human-friendly elapsed rendering — `12ms` / `340ms` / `1.4s` /
|
||||
/// `12.3s` / `4m30s`. The structured `elapsed_ms` field still carries
|
||||
/// the raw millisecond number for log aggregators.
|
||||
fn format_elapsed(ms: u128) -> String {
|
||||
if ms < 1000 {
|
||||
format!("{}ms", ms)
|
||||
} else if ms < 60_000 {
|
||||
format!("{:.1}s", (ms as f64) / 1000.0)
|
||||
} else {
|
||||
let secs = ms / 1000;
|
||||
let m = secs / 60;
|
||||
let s = secs % 60;
|
||||
format!("{}m{}s", m, s)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -361,8 +385,7 @@ mod tests {
|
||||
let registry = Arc::new(JobRegistry::new());
|
||||
registry
|
||||
.register(handler, Some(Duration::from_millis(100)), None)
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
let entry = registry.get("overrun").await.unwrap();
|
||||
|
||||
// Kick off dispatch 1 in the background — it holds the permit
|
||||
@@ -402,8 +425,7 @@ mod tests {
|
||||
Some(Duration::from_millis(100)),
|
||||
Some(Duration::from_millis(50)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
let entry = registry.get("slow").await.unwrap();
|
||||
|
||||
dispatch("slow", entry.clone(), &JobRunArgs::default()).await;
|
||||
|
||||
@@ -24,10 +24,18 @@
|
||||
|
||||
mod engine;
|
||||
mod handler;
|
||||
mod pg_job_store;
|
||||
mod recoverable;
|
||||
mod registry;
|
||||
mod types;
|
||||
|
||||
pub use engine::SchedulerEngine;
|
||||
pub use handler::JobHandler;
|
||||
pub use pg_job_store::{PgJobStore, PgJobStoreProvider};
|
||||
pub use recoverable::{
|
||||
Finding, JobStore, JobStoreProvider, OpenedRun, ProgressKind, RecoverableAdapter,
|
||||
RecoverableJobHandler, RunOutcome, RunProgress, RunStatus, RunSummary, derive_progress,
|
||||
record_or_log, run_or_resume,
|
||||
};
|
||||
pub use registry::{JobEntry, JobRegistry, JobSummary, RegisterError};
|
||||
pub use types::{ErrCause, JobOutcome, JobRunArgs};
|
||||
|
||||
@@ -0,0 +1,713 @@
|
||||
//! PostgreSQL adapter for the recoverable-run engine
|
||||
//! ([`super::recoverable`]). Concrete impls of [`JobStore`] and
|
||||
//! [`JobStoreProvider`] backed by `jobs.recoverable_runs`.
|
||||
//!
|
||||
//! Both types are cheap to construct (just an `Arc<PgPool>` plus, for
|
||||
//! `PgJobStore`, the bound run's id + started_at). One `PgJobStoreProvider`
|
||||
//! lives on `AppState.core.job_store_provider`; per-run `PgJobStore`
|
||||
//! instances are built by `open_or_start`.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::common::errors::DomainError;
|
||||
|
||||
use super::recoverable::{
|
||||
Finding, JobStore, JobStoreProvider, OpenedRun, ProgressKind, RunStatus, RunSummary,
|
||||
derive_progress,
|
||||
};
|
||||
|
||||
// ─── PgJobStore — bound to one run ──────────────────────────────────────────
|
||||
|
||||
/// A `JobStore` bound to a specific `jobs.recoverable_runs.id`. Every
|
||||
/// method issues one small UPDATE / SELECT against that row.
|
||||
pub struct PgJobStore {
|
||||
pool: Arc<PgPool>,
|
||||
run_id: Uuid,
|
||||
started_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl PgJobStore {
|
||||
/// Called only from [`PgJobStoreProvider::open_or_start`] and its
|
||||
/// test helpers — implementors never construct one directly.
|
||||
pub(super) fn new(pool: Arc<PgPool>, run_id: Uuid, started_at: DateTime<Utc>) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
run_id,
|
||||
started_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_sqlx_err(op: &'static str, e: sqlx::Error) -> DomainError {
|
||||
DomainError::internal_error("JobStore", format!("{op}: {e}"))
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobStore for PgJobStore {
|
||||
fn run_id(&self) -> Uuid {
|
||||
self.run_id
|
||||
}
|
||||
|
||||
fn started_at(&self) -> DateTime<Utc> {
|
||||
self.started_at
|
||||
}
|
||||
|
||||
async fn status(&self) -> Result<RunStatus, DomainError> {
|
||||
let row: Option<(String,)> =
|
||||
sqlx::query_as("SELECT status FROM jobs.recoverable_runs WHERE id = $1")
|
||||
.bind(self.run_id)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("status", e))?;
|
||||
let raw = row
|
||||
.ok_or_else(|| {
|
||||
DomainError::internal_error("JobStore", format!("run vanished: {}", self.run_id))
|
||||
})?
|
||||
.0;
|
||||
RunStatus::parse(&raw).ok_or_else(|| {
|
||||
DomainError::internal_error("JobStore", format!("unknown status value: {raw}"))
|
||||
})
|
||||
}
|
||||
|
||||
async fn checkpoint(&self, cursor: Vec<u8>, delta_count: u64) -> Result<(), DomainError> {
|
||||
// stats.scanned_count += delta_count. jsonb_set expects the new
|
||||
// value serialised as jsonb; the cast chain from bigint → text
|
||||
// → jsonb is the standard way to bump a numeric counter without
|
||||
// pulling the whole JSONB into Rust.
|
||||
let delta = delta_count as i64;
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET cursor = $2,
|
||||
stats = jsonb_set(
|
||||
stats,
|
||||
'{scanned_count}',
|
||||
((COALESCE(stats->>'scanned_count', '0')::bigint + $3)::text)::jsonb
|
||||
),
|
||||
last_progress_at = NOW()
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(self.run_id)
|
||||
.bind(&cursor[..])
|
||||
.bind(delta)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("checkpoint", e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn record_finding(
|
||||
&self,
|
||||
kind: &str,
|
||||
severity: &str,
|
||||
resource_id: Option<Uuid>,
|
||||
detail: serde_json::Value,
|
||||
) -> Result<(), DomainError> {
|
||||
// Two writes in one round-trip via CTE: INSERT the finding
|
||||
// row + UPDATE stats.finding_count on the parent run. The
|
||||
// counter stays a coarse UI hint — the source of truth is
|
||||
// the `jobs.run_findings` table itself. Even if the counter
|
||||
// drifts (crash mid-statement, hand-edited row), aggregations
|
||||
// stay accurate. Bumping in the same statement avoids two
|
||||
// round trips per finding on a hot scan; on a normal
|
||||
// consistency run the ratio of findings-to-batches is low
|
||||
// enough that a round trip either way is fine, but this shape
|
||||
// scales to a bulk-finding tenant without change.
|
||||
//
|
||||
// `resource_id` is nullable in the schema; when None here we
|
||||
// bind `Option::<Uuid>::None` and sqlx encodes it as SQL NULL.
|
||||
let bumped = sqlx::query(
|
||||
r#"
|
||||
WITH inserted AS (
|
||||
INSERT INTO jobs.run_findings
|
||||
(run_id, kind, severity, resource_id, detail)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING run_id
|
||||
)
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET stats = jsonb_set(
|
||||
stats,
|
||||
'{finding_count}',
|
||||
((COALESCE(stats->>'finding_count', '0')::bigint + 1)::text)::jsonb
|
||||
)
|
||||
WHERE id = (SELECT run_id FROM inserted)
|
||||
"#,
|
||||
)
|
||||
.bind(self.run_id)
|
||||
.bind(kind)
|
||||
.bind(severity)
|
||||
.bind(resource_id)
|
||||
.bind(&detail)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("record_finding", e))?;
|
||||
|
||||
// A rows_affected == 0 on the UPDATE would mean the parent run
|
||||
// row vanished between INSERT and UPDATE — theoretically
|
||||
// impossible under our CASCADE FK (deleting the run drops the
|
||||
// finding), so we don't error, but a debug log covers the
|
||||
// defensive path.
|
||||
if bumped.rows_affected() == 0 {
|
||||
tracing::debug!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "record_finding.counter_bump_noop",
|
||||
run_id = %self.run_id,
|
||||
"record_finding: parent run row missing during counter bump"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn seed_progress_params(
|
||||
&self,
|
||||
total: u64,
|
||||
kind: ProgressKind,
|
||||
) -> Result<(), DomainError> {
|
||||
// Stamp `params.total_rows` + `params.progress_kind` in one
|
||||
// UPDATE. Two `jsonb_set` calls compose left-to-right so both
|
||||
// keys land atomically. `bigint` cast handles the (theoretical)
|
||||
// > 2^31 subject-row case.
|
||||
let total_i64 = total as i64;
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET params = jsonb_set(
|
||||
jsonb_set(
|
||||
params,
|
||||
'{total_rows}',
|
||||
to_jsonb($2::bigint)
|
||||
),
|
||||
'{progress_kind}',
|
||||
to_jsonb($3::text)
|
||||
)
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(self.run_id)
|
||||
.bind(total_i64)
|
||||
.bind(kind.as_str())
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("seed_progress_params", e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_completed(&self) -> Result<(), DomainError> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET status = 'Completed',
|
||||
completed_at = NOW(),
|
||||
last_progress_at = NOW()
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(self.run_id)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("mark_completed", e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_paused(&self, cursor: Option<Vec<u8>>) -> Result<(), DomainError> {
|
||||
// Two-query variant would be simpler but this preserves the
|
||||
// final cursor value in one statement whether or not the
|
||||
// handler advanced it since the last checkpoint.
|
||||
if let Some(c) = cursor {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET status = 'Paused',
|
||||
cursor = $2,
|
||||
last_progress_at = NOW()
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(self.run_id)
|
||||
.bind(&c[..])
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("mark_paused", e))?;
|
||||
} else {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET status = 'Paused',
|
||||
last_progress_at = NOW()
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(self.run_id)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("mark_paused", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn mark_failed(&self, message: &str) -> Result<(), DomainError> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET status = 'Failed',
|
||||
completed_at = NOW(),
|
||||
last_progress_at = NOW(),
|
||||
error_message = $2
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(self.run_id)
|
||||
.bind(message)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("mark_failed", e))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ─── PgJobStoreProvider — registry-level ops ────────────────────────────────
|
||||
|
||||
/// The `JobStoreProvider` PG-backed implementation. Constructs
|
||||
/// `PgJobStore` handles via `open_or_start`, and drives the boot-time
|
||||
/// crash-recovery sweep.
|
||||
pub struct PgJobStoreProvider {
|
||||
pool: Arc<PgPool>,
|
||||
}
|
||||
|
||||
impl PgJobStoreProvider {
|
||||
pub fn new(pool: Arc<PgPool>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobStoreProvider for PgJobStoreProvider {
|
||||
async fn open_or_start(&self, job_name: &str) -> Result<OpenedRun, DomainError> {
|
||||
// Two-shot: look up the latest non-terminal row; if found,
|
||||
// dispatch on its status; if not, INSERT a fresh Running row.
|
||||
//
|
||||
// Concurrent-insert race is caught by the partial unique index
|
||||
// `one_active_run_per_job` — the losing INSERT falls back to
|
||||
// re-querying and dispatching on whatever the winner wrote.
|
||||
// We retry once because after the first losing insert, the
|
||||
// winning row is guaranteed to exist and no third caller can
|
||||
// race in ahead of us (they'd hit the same unique index).
|
||||
for attempt in 0..2 {
|
||||
match self.try_open_or_start(job_name).await {
|
||||
Ok(opened) => return Ok(opened),
|
||||
Err(OpenErr::Retry) => {
|
||||
tracing::debug!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "recoverable.open_or_start.race",
|
||||
job = job_name,
|
||||
attempt = attempt,
|
||||
"open_or_start lost to a concurrent INSERT; retrying"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
Err(OpenErr::Fatal(e)) => return Err(e),
|
||||
}
|
||||
}
|
||||
Err(DomainError::internal_error(
|
||||
"JobStore",
|
||||
format!("open_or_start({job_name}): retry budget exhausted"),
|
||||
))
|
||||
}
|
||||
|
||||
async fn boot_recovery_sweep(&self) -> Result<u64, DomainError> {
|
||||
// Every row abandoned in Running / CancelRequested by the
|
||||
// previous process flips to Paused with a synthetic
|
||||
// error_message. We DO NOT auto-resume — operators trigger
|
||||
// the resume explicitly per the trait doc.
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET status = 'Paused',
|
||||
error_message = COALESCE(error_message, 'server restart mid-run')
|
||||
WHERE status IN ('Running', 'CancelRequested')
|
||||
"#,
|
||||
)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("boot_recovery_sweep", e))?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
async fn list_runs(&self, job_name: &str, limit: u32) -> Result<Vec<RunSummary>, DomainError> {
|
||||
// Cap the limit at 100 defensively — the API layer should
|
||||
// also clamp, but a broken caller shouldn't tank the DB.
|
||||
let capped = limit.min(100) as i64;
|
||||
let rows: Vec<RunSummaryRow> = sqlx::query_as(RUN_SUMMARY_SELECT_LIST)
|
||||
.bind(job_name)
|
||||
.bind(capped)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("list_runs", e))?;
|
||||
rows.into_iter().map(row_to_summary).collect()
|
||||
}
|
||||
|
||||
async fn get_run_by_id(&self, run_id: Uuid) -> Result<Option<RunSummary>, DomainError> {
|
||||
let row: Option<RunSummaryRow> = sqlx::query_as(RUN_SUMMARY_SELECT_BY_ID)
|
||||
.bind(run_id)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("get_run_by_id", e))?;
|
||||
row.map(row_to_summary).transpose()
|
||||
}
|
||||
|
||||
async fn list_findings(
|
||||
&self,
|
||||
run_id: Uuid,
|
||||
limit: u32,
|
||||
offset: u32,
|
||||
) -> Result<Vec<Finding>, DomainError> {
|
||||
let capped = limit.min(500) as i64;
|
||||
let off = offset as i64;
|
||||
let rows: Vec<(
|
||||
Uuid,
|
||||
Uuid,
|
||||
String,
|
||||
String,
|
||||
Option<Uuid>,
|
||||
serde_json::Value,
|
||||
DateTime<Utc>,
|
||||
)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT id, run_id, kind, severity, resource_id, detail, created_at
|
||||
FROM jobs.run_findings
|
||||
WHERE run_id = $1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(run_id)
|
||||
.bind(capped)
|
||||
.bind(off)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("list_findings", e))?;
|
||||
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(
|
||||
|(id, run_id, kind, severity, resource_id, detail, created_at)| Finding {
|
||||
id,
|
||||
run_id,
|
||||
kind,
|
||||
severity,
|
||||
resource_id,
|
||||
detail,
|
||||
created_at,
|
||||
},
|
||||
)
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn finding_severity_counts(
|
||||
&self,
|
||||
run_id: Uuid,
|
||||
) -> Result<Vec<(String, u64)>, DomainError> {
|
||||
let rows: Vec<(String, i64)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT severity, COUNT(*)::bigint
|
||||
FROM jobs.run_findings
|
||||
WHERE run_id = $1
|
||||
GROUP BY severity
|
||||
"#,
|
||||
)
|
||||
.bind(run_id)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("finding_severity_counts", e))?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(sev, count)| (sev, count.max(0) as u64))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn purge_terminal_runs(&self, retention_days: i32) -> Result<u64, DomainError> {
|
||||
// Defensive floor — zero would eat just-completed runs;
|
||||
// negative would eat the whole terminal history.
|
||||
let days = retention_days.max(1);
|
||||
// `ON DELETE CASCADE` on jobs.run_findings.run_id
|
||||
// (migration 20260930000001) drops findings with their
|
||||
// parent run. Non-terminal statuses (Running / Paused /
|
||||
// CancelRequested) explicitly excluded to protect
|
||||
// in-flight work.
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
DELETE FROM jobs.recoverable_runs
|
||||
WHERE status IN ('Completed', 'Failed')
|
||||
AND completed_at IS NOT NULL
|
||||
AND completed_at < NOW() - ($1 || ' days')::interval
|
||||
"#,
|
||||
)
|
||||
.bind(days.to_string())
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("purge_terminal_runs", e))?;
|
||||
Ok(result.rows_affected())
|
||||
}
|
||||
|
||||
async fn request_cancel(&self, job_name: &str) -> Result<Option<Uuid>, DomainError> {
|
||||
// Only Running → CancelRequested flips. `Paused` can be
|
||||
// cancelled by not resuming — no need for a state change.
|
||||
// `CancelRequested` already is what it is.
|
||||
// Multiple Running rows shouldn't exist (partial unique index),
|
||||
// but LIMIT 1 is defensive.
|
||||
let flipped: Option<(Uuid,)> = sqlx::query_as(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET status = 'CancelRequested',
|
||||
last_progress_at = NOW()
|
||||
WHERE id = (
|
||||
SELECT id FROM jobs.recoverable_runs
|
||||
WHERE job_name = $1
|
||||
AND status = 'Running'
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id
|
||||
"#,
|
||||
)
|
||||
.bind(job_name)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| map_sqlx_err("request_cancel", e))?;
|
||||
Ok(flipped.map(|(id,)| id))
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Shared row → RunSummary decoder ────────────────────────────────────────
|
||||
|
||||
/// Row shape returned by the run-summary SELECTs. Kept as a distinct
|
||||
/// type so both `list_runs` and `get_run_by_id` share the projection
|
||||
/// (SQL column list + decoder). Order matches the SELECT below.
|
||||
type RunSummaryRow = (
|
||||
Uuid, // id
|
||||
String, // job_name
|
||||
String, // status
|
||||
DateTime<Utc>, // started_at
|
||||
DateTime<Utc>, // last_progress_at
|
||||
Option<DateTime<Utc>>, // completed_at
|
||||
Option<Vec<u8>>, // cursor
|
||||
serde_json::Value, // stats
|
||||
serde_json::Value, // params
|
||||
Option<String>, // error_message
|
||||
);
|
||||
|
||||
const RUN_SUMMARY_COLUMNS: &str = "id, job_name, status, started_at, last_progress_at, completed_at, cursor, stats, params, error_message";
|
||||
|
||||
// `format!` isn't const, but `concat!` gives us a &'static str at compile
|
||||
// time — worth it so the SELECT strings show up in tracing / SQL logs
|
||||
// as one contiguous line instead of a runtime string build.
|
||||
const RUN_SUMMARY_SELECT_LIST: &str = concat!(
|
||||
"SELECT id, job_name, status, started_at, last_progress_at, completed_at, cursor, stats, params, error_message ",
|
||||
"FROM jobs.recoverable_runs ",
|
||||
"WHERE job_name = $1 ",
|
||||
"ORDER BY started_at DESC ",
|
||||
"LIMIT $2"
|
||||
);
|
||||
|
||||
const RUN_SUMMARY_SELECT_BY_ID: &str = concat!(
|
||||
"SELECT id, job_name, status, started_at, last_progress_at, completed_at, cursor, stats, params, error_message ",
|
||||
"FROM jobs.recoverable_runs ",
|
||||
"WHERE id = $1"
|
||||
);
|
||||
|
||||
fn row_to_summary(row: RunSummaryRow) -> Result<RunSummary, DomainError> {
|
||||
let (
|
||||
id,
|
||||
job_name,
|
||||
status_str,
|
||||
started_at,
|
||||
last_progress_at,
|
||||
completed_at,
|
||||
cursor,
|
||||
stats,
|
||||
params,
|
||||
error_message,
|
||||
) = row;
|
||||
let status = RunStatus::parse(&status_str).ok_or_else(|| {
|
||||
DomainError::internal_error("JobStore", format!("unknown status: {status_str}"))
|
||||
})?;
|
||||
|
||||
// Derive progress from stats.scanned_count + params.total_rows +
|
||||
// params.progress_kind. All three are optional — if the tenant
|
||||
// didn't seed a total (no count_total override) the block is None
|
||||
// and the UI hides the bar. `derive_progress` also guards against
|
||||
// total = 0 (empty-subject run — bar would be meaningless).
|
||||
let scanned = stats
|
||||
.get("scanned_count")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
let total = params.get("total_rows").and_then(|v| v.as_u64());
|
||||
let kind = params
|
||||
.get("progress_kind")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(ProgressKind::parse);
|
||||
let progress = derive_progress(scanned, total, kind);
|
||||
|
||||
Ok(RunSummary {
|
||||
id,
|
||||
job_name,
|
||||
status,
|
||||
started_at,
|
||||
last_progress_at,
|
||||
completed_at,
|
||||
stats,
|
||||
params,
|
||||
cursor_hex: cursor.map(hex::encode),
|
||||
error_message,
|
||||
progress,
|
||||
})
|
||||
}
|
||||
|
||||
// Suppress the dead-code lint on the column list — kept as a
|
||||
// human-readable constant even though the actual SELECTs currently
|
||||
// inline it. Future rewrites of the SELECTs (e.g. adding stats
|
||||
// projection) will use it.
|
||||
#[allow(dead_code)]
|
||||
const _RUN_SUMMARY_COLUMNS_UNUSED: &str = RUN_SUMMARY_COLUMNS;
|
||||
|
||||
/// Row shape returned by `open_or_start`'s SELECT — factored out
|
||||
/// so clippy's `type_complexity` lint doesn't yell at the query.
|
||||
type ExistingRun = (Uuid, String, DateTime<Utc>, Option<Vec<u8>>);
|
||||
|
||||
/// Internal error surface for the two-shot open_or_start retry loop.
|
||||
enum OpenErr {
|
||||
/// Lost to a concurrent INSERT — caller retries.
|
||||
Retry,
|
||||
/// Any other DB error — surfaces to caller unchanged.
|
||||
Fatal(DomainError),
|
||||
}
|
||||
|
||||
impl PgJobStoreProvider {
|
||||
/// One attempt of open_or_start. Returns `Err(Retry)` on the
|
||||
/// unique-index-conflict path so the outer loop re-queries.
|
||||
async fn try_open_or_start(&self, job_name: &str) -> Result<OpenedRun, OpenErr> {
|
||||
// Latest non-terminal row for this job_name, if any.
|
||||
let existing: Option<ExistingRun> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT id, status, started_at, cursor
|
||||
FROM jobs.recoverable_runs
|
||||
WHERE job_name = $1
|
||||
AND status IN ('Running', 'Paused', 'CancelRequested')
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.bind(job_name)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| OpenErr::Fatal(map_sqlx_err("open_or_start.select", e)))?;
|
||||
|
||||
match existing {
|
||||
Some((id, raw_status, _started_at, _cursor)) => {
|
||||
let status = RunStatus::parse(&raw_status).ok_or_else(|| {
|
||||
OpenErr::Fatal(DomainError::internal_error(
|
||||
"JobStore",
|
||||
format!("unknown status: {raw_status}"),
|
||||
))
|
||||
})?;
|
||||
match status {
|
||||
RunStatus::Running | RunStatus::CancelRequested => {
|
||||
Ok(OpenedRun::AlreadyActive { run_id: id, status })
|
||||
}
|
||||
RunStatus::Paused => {
|
||||
// Flip to Running and hand back the cursor.
|
||||
// Race note: another concurrent caller could
|
||||
// race the same UPDATE. Both would succeed
|
||||
// (Paused → Running is idempotent), but only
|
||||
// one caller's dispatch would then race the
|
||||
// partial unique index on subsequent
|
||||
// operations. Acceptable — the loser's
|
||||
// handler will observe `status = Running`
|
||||
// (via `store.status()`) and can early-exit.
|
||||
// In practice this is a rare edge case that
|
||||
// ONLY hits if two admin triggers land in
|
||||
// the same microsecond.
|
||||
let row: Option<(DateTime<Utc>, Option<Vec<u8>>)> = sqlx::query_as(
|
||||
r#"
|
||||
UPDATE jobs.recoverable_runs
|
||||
SET status = 'Running',
|
||||
last_progress_at = NOW()
|
||||
WHERE id = $1
|
||||
RETURNING started_at, cursor
|
||||
"#,
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| OpenErr::Fatal(map_sqlx_err("open_or_start.resume", e)))?;
|
||||
let (started_at, cursor_bytes) = row.ok_or_else(|| {
|
||||
OpenErr::Fatal(DomainError::internal_error(
|
||||
"JobStore",
|
||||
format!("run vanished during resume: {id}"),
|
||||
))
|
||||
})?;
|
||||
let store: Arc<dyn JobStore> =
|
||||
Arc::new(PgJobStore::new(self.pool.clone(), id, started_at));
|
||||
Ok(OpenedRun::Resumed {
|
||||
store,
|
||||
cursor: cursor_bytes.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
// Terminal states shouldn't appear here (WHERE
|
||||
// clause filters them). Defensive branch.
|
||||
_ => Err(OpenErr::Fatal(DomainError::internal_error(
|
||||
"JobStore",
|
||||
format!("terminal status leaked into open_or_start: {status:?}"),
|
||||
))),
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// No non-terminal row → INSERT a fresh one. The
|
||||
// partial unique index protects against a concurrent
|
||||
// second INSERT; on conflict we retry.
|
||||
let run_id = Uuid::new_v4();
|
||||
let now = Utc::now();
|
||||
// ON CONFLICT here infers the partial unique index by
|
||||
// matching `(job_name)` + the WHERE predicate that
|
||||
// matches `one_active_run_per_job`. We cannot use
|
||||
// `ON CONFLICT ON CONSTRAINT one_active_run_per_job`
|
||||
// because `CREATE UNIQUE INDEX` produces an index, not
|
||||
// a named constraint from PG's perspective; that
|
||||
// syntax is reserved for `ALTER TABLE ... ADD CONSTRAINT
|
||||
// UNIQUE`. Inference form is equivalent and works with
|
||||
// partial indexes.
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO jobs.recoverable_runs
|
||||
(id, job_name, status, started_at, last_progress_at)
|
||||
VALUES ($1, $2, 'Running', $3, $3)
|
||||
ON CONFLICT (job_name)
|
||||
WHERE status IN ('Running', 'Paused', 'CancelRequested')
|
||||
DO NOTHING
|
||||
"#,
|
||||
)
|
||||
.bind(run_id)
|
||||
.bind(job_name)
|
||||
.bind(now)
|
||||
.execute(self.pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| OpenErr::Fatal(map_sqlx_err("open_or_start.insert", e)))?;
|
||||
|
||||
if result.rows_affected() == 1 {
|
||||
let store: Arc<dyn JobStore> =
|
||||
Arc::new(PgJobStore::new(self.pool.clone(), run_id, now));
|
||||
Ok(OpenedRun::Fresh { store })
|
||||
} else {
|
||||
// Someone raced us. Retry to pick up their row.
|
||||
Err(OpenErr::Retry)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -69,10 +69,7 @@ impl JobRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a job. Returns an error if a job with the same name
|
||||
/// is already registered — names are the primary identifier
|
||||
/// everywhere (logs, admin URLs, env vars) and collisions would
|
||||
/// hide bugs.
|
||||
/// Register a job — production wiring path.
|
||||
///
|
||||
/// - `interval = Some(dur)` → **scheduled**. The supervisor fires
|
||||
/// the job every `dur`, starting `now + dur`. Registration does
|
||||
@@ -83,11 +80,59 @@ impl JobRegistry {
|
||||
/// fires this job. Admin endpoint (or programmatic callers) can
|
||||
/// still invoke it via [`JobRegistry::trigger`] — the dispatch
|
||||
/// goes through the same panic/timeout/exclusivity gates.
|
||||
///
|
||||
/// **Panics on error.** Registration failure (duplicate name today)
|
||||
/// is a DI-wiring bug — the server must not start with a mis-wired
|
||||
/// scheduler. Emits a uniform `job.registered` log line on success
|
||||
/// so callers don't reinvent the log message at every site.
|
||||
///
|
||||
/// For unit tests that need to assert the error path use
|
||||
/// [`Self::try_register`] instead.
|
||||
pub async fn register(
|
||||
&self,
|
||||
handler: Arc<dyn JobHandler>,
|
||||
interval: Option<Duration>,
|
||||
timeout: Option<Duration>,
|
||||
) {
|
||||
let name = handler.name().to_string();
|
||||
match self.try_register(handler, interval, timeout).await {
|
||||
Ok(()) => {
|
||||
let cadence = match interval {
|
||||
Some(dur) => {
|
||||
let secs = dur.as_secs();
|
||||
if secs % 3600 == 0 {
|
||||
format!("every {} h", secs / 3600)
|
||||
} else if secs % 60 == 0 {
|
||||
format!("every {} min", secs / 60)
|
||||
} else {
|
||||
format!("every {} s", secs)
|
||||
}
|
||||
}
|
||||
None => "on-demand".to_string(),
|
||||
};
|
||||
tracing::info!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "job.registered",
|
||||
job = %name,
|
||||
cadence = %cadence,
|
||||
"job {} registered ({})",
|
||||
name,
|
||||
cadence,
|
||||
);
|
||||
}
|
||||
Err(e) => panic!("JobRegistry::register({name}) failed — DI wiring bug: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fallible sibling of [`Self::register`]. Returns `Err` on
|
||||
/// duplicate-name instead of panicking, and does NOT emit the
|
||||
/// `job.registered` log line — for unit tests that need to
|
||||
/// assert failure without triggering the boot panic path.
|
||||
pub async fn try_register(
|
||||
&self,
|
||||
handler: Arc<dyn JobHandler>,
|
||||
interval: Option<Duration>,
|
||||
timeout: Option<Duration>,
|
||||
) -> Result<(), RegisterError> {
|
||||
let name = handler.name().to_string();
|
||||
let mut guard = self.entries.write().await;
|
||||
@@ -286,11 +331,9 @@ mod tests {
|
||||
async fn register_and_pick_next() {
|
||||
let reg = JobRegistry::new();
|
||||
reg.register(handler("job_a"), Some(Duration::from_secs(60)), None)
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
reg.register(handler("job_b"), Some(Duration::from_secs(10)), None)
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
|
||||
let (next_name, _) = reg.pick_next().await.expect("expected a due job");
|
||||
// job_b has the shorter interval → earlier next_run_at.
|
||||
@@ -300,16 +343,30 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn duplicate_registration_rejected() {
|
||||
let reg = JobRegistry::new();
|
||||
reg.register(handler("job_x"), Some(Duration::from_secs(60)), None)
|
||||
// Use the fallible `try_register` here so we can assert the
|
||||
// Err path without triggering `register`'s boot-time panic.
|
||||
reg.try_register(handler("job_x"), Some(Duration::from_secs(60)), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let err = reg
|
||||
.register(handler("job_x"), Some(Duration::from_secs(60)), None)
|
||||
.try_register(handler("job_x"), Some(Duration::from_secs(60)), None)
|
||||
.await
|
||||
.expect_err("duplicate name must be rejected");
|
||||
assert!(matches!(err, RegisterError::DuplicateName(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[should_panic(expected = "DI wiring bug")]
|
||||
async fn register_panics_on_duplicate() {
|
||||
let reg = JobRegistry::new();
|
||||
reg.register(handler("job_dup"), Some(Duration::from_secs(60)), None)
|
||||
.await;
|
||||
// Second register with same name — boot panic. Anything doing
|
||||
// this outside a #[should_panic] test is a mis-wired DI.
|
||||
reg.register(handler("job_dup"), Some(Duration::from_secs(60)), None)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_registry_picks_nothing() {
|
||||
let reg = JobRegistry::new();
|
||||
@@ -320,11 +377,9 @@ mod tests {
|
||||
async fn snapshot_all_returns_every_entry() {
|
||||
let reg = JobRegistry::new();
|
||||
reg.register(handler("a"), Some(Duration::from_secs(1)), None)
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
reg.register(handler("b"), Some(Duration::from_secs(1)), None)
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
let all = reg.snapshot_all().await;
|
||||
assert_eq!(all.len(), 2);
|
||||
}
|
||||
@@ -334,12 +389,9 @@ mod tests {
|
||||
let reg = JobRegistry::new();
|
||||
// Scheduled job with a long interval.
|
||||
reg.register(handler("scheduled"), Some(Duration::from_secs(3600)), None)
|
||||
.await
|
||||
.unwrap();
|
||||
.await;
|
||||
// On-demand job — supervisor must never pick it.
|
||||
reg.register(handler("on_demand"), None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
reg.register(handler("on_demand"), None, None).await;
|
||||
|
||||
let (next_name, _) = reg.pick_next().await.expect("scheduled job due");
|
||||
assert_eq!(
|
||||
@@ -351,7 +403,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn trigger_dispatches_on_demand_job() {
|
||||
let reg = Arc::new(JobRegistry::new());
|
||||
reg.register(handler("gc"), None, None).await.unwrap();
|
||||
reg.register(handler("gc"), None, None).await;
|
||||
|
||||
let outcome = reg
|
||||
.trigger("gc", &JobRunArgs::default())
|
||||
|
||||
@@ -25,9 +25,19 @@ use serde::{Deserialize, Serialize};
|
||||
/// - `dedup_gc` — skip the orphan grace window (grace = 0).
|
||||
/// - `grant_cleanup` — grace = 0.
|
||||
/// - Others (trash_cleanup, storage_reconcile, …) — ignored.
|
||||
///
|
||||
/// Semantics of `deep`, per job:
|
||||
/// - `consistency_batch` — propagate to sub-jobs; only `storage_consistency`
|
||||
/// currently respects it. Wraps the "run all consistency checks
|
||||
/// including the slow ones" case behind the same job_name lock as
|
||||
/// the normal batch (Ed's Option B, 2026-07-29).
|
||||
/// - `storage_consistency` (future) — enables per-blob re-BLAKE3 (bitrot
|
||||
/// detection) + mime sniff alongside the fast orphan check.
|
||||
/// - Others — ignored.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct JobRunArgs {
|
||||
pub force: bool,
|
||||
pub deep: bool,
|
||||
}
|
||||
|
||||
/// Uniform outcome the supervisor logs and stores for every job dispatch.
|
||||
|
||||
@@ -389,4 +389,14 @@ impl BlobStorageBackend for AzureBlobBackend {
|
||||
fn local_blob_path(&self, _hash: &str) -> Option<PathBuf> {
|
||||
None
|
||||
}
|
||||
|
||||
// TODO: implement `list_blob_hashes` via
|
||||
// `container_client.list_blobs()` (`azure_storage_blobs`
|
||||
// paginator). Same filter as local + S3 impls:
|
||||
// `<xx>/<64-hex>.blob` naming. Currently inherits the trait
|
||||
// default which returns `operation_not_supported` — the
|
||||
// `backend_consistency` tenant handles that by emitting a
|
||||
// single run-level `backend_unenumerable` finding and
|
||||
// completing without per-blob probes. Ship as a follow-up once
|
||||
// there's an Azure test environment to validate against.
|
||||
}
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
//! Fifth tenant of Part 2 (recoverable-run engine).
|
||||
//!
|
||||
//! Iterates the storage backend's blob-enumeration surface and
|
||||
//! reports every blob physically present on the backend that has NO
|
||||
//! matching row in `storage.blobs`. Complements
|
||||
//! `blobs_consistency` (which walks the DB and probes the backend):
|
||||
//! together they close the reference graph.
|
||||
//!
|
||||
//! ### Per-row check
|
||||
//!
|
||||
//! * `orphan_blob` (severity `inconsistent`) — bytes on disk / S3 /
|
||||
//! Azure with no registry row. Not data-loss (nothing broken —
|
||||
//! just storage overhead), but points at dedup_gc or
|
||||
//! ingest-path drift. Recovery = register-registry-row (if the
|
||||
//! bytes are still needed) OR delete the file (if truly orphan).
|
||||
//!
|
||||
//! ### Run-level check
|
||||
//!
|
||||
//! * `backend_unenumerable` (severity `anomaly`) — the backend
|
||||
//! returned `operation_not_supported` on the first
|
||||
//! `list_blob_hashes` call. Currently this fires when a
|
||||
//! `MigrationBlobBackend` is active (refuses enumeration
|
||||
//! mid-migration by design) or on an Azure backend (Azure impl
|
||||
//! deferred). Informational — operators know they can't rely on
|
||||
//! this scan under that config.
|
||||
//!
|
||||
//! ### Grace window
|
||||
//!
|
||||
//! Skip orphans whose backend mtime is within the last hour. Same
|
||||
//! shape as `blobs_consistency` + `dedup_gc`: matches the
|
||||
//! durability-before-visibility gap in the write path.
|
||||
//!
|
||||
//! ### Cost profile
|
||||
//!
|
||||
//! Batched: fetch N hashes from the backend, do one
|
||||
//! `WHERE hash = ANY($1)` DB probe per batch, set-difference in
|
||||
//! Rust. Dedup savings: yes — a chunk shared by 5 files still
|
||||
//! walks once. On local backend the walk is
|
||||
//! `walkdir + fs::metadata` per file (fast). On S3 the walk is
|
||||
//! `ListObjectsV2` (rate-limited but paginated). Progress bar
|
||||
//! uses `COUNT(*) FROM storage.blobs` as the approximate
|
||||
//! denominator (backend count ≈ blob count on a healthy install;
|
||||
//! deviation IS the finding).
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{Duration, Utc};
|
||||
use sqlx::PgPool;
|
||||
|
||||
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||
use crate::infrastructure::scheduler::{
|
||||
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, ProgressKind, RecoverableJobHandler,
|
||||
RunOutcome, RunStatus, record_or_log,
|
||||
};
|
||||
|
||||
pub const BACKEND_CONSISTENCY_JOB_NAME: &str = "backend_consistency";
|
||||
|
||||
/// Batch size for backend enumeration + DB probe. 500 is enough to
|
||||
/// amortise the DB round-trip while keeping the cancel-poll cadence
|
||||
/// sub-second (each batch = one backend list + one DB probe + Rust
|
||||
/// set-difference). Larger batches on S3 hit ListObjectsV2's
|
||||
/// per-request limit (1000) with wasted rows filtered client-side;
|
||||
/// smaller batches over-poll the DB.
|
||||
const BATCH_SIZE: usize = 500;
|
||||
|
||||
/// Grace window — orphans younger than this are skipped, since the
|
||||
/// write path is durability-before-visibility: bytes hit disk before
|
||||
/// the `storage.blobs` row is inserted. A scan catching a blob
|
||||
/// mid-write would false-positive it as orphan. Matches
|
||||
/// `blobs_consistency` + `dedup_gc`.
|
||||
const CREATE_GRACE: Duration = Duration::hours(1);
|
||||
|
||||
/// Cap on affected-blob examples surfaced in the run-level
|
||||
/// `backend_unenumerable` finding. Keeps the finding detail bounded.
|
||||
const _MAX_EXAMPLES: usize = 5;
|
||||
|
||||
pub struct BackendConsistencyCheck {
|
||||
pool: Arc<PgPool>,
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
}
|
||||
|
||||
impl BackendConsistencyCheck {
|
||||
pub fn new(pool: Arc<PgPool>, backend: Arc<dyn BlobStorageBackend>) -> Self {
|
||||
Self { pool, backend }
|
||||
}
|
||||
|
||||
pub async fn register_recoverable_job(
|
||||
self: Arc<Self>,
|
||||
registry: &JobRegistry,
|
||||
provider: &Arc<dyn JobStoreProvider>,
|
||||
) -> Arc<Self> {
|
||||
registry
|
||||
.register_recoverable_job(self.clone(), provider.clone(), None)
|
||||
.await;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RecoverableJobHandler for BackendConsistencyCheck {
|
||||
fn name(&self) -> &str {
|
||||
BACKEND_CONSISTENCY_JOB_NAME
|
||||
}
|
||||
|
||||
/// Approximate total: on a healthy install every backend blob
|
||||
/// has a `storage.blobs` row, so the DB count is a proxy for
|
||||
/// the backend count. The fraction deviating from 1.0 at run
|
||||
/// end IS informative — a fraction of 1.05 means the backend
|
||||
/// holds ~5% orphan bytes, which is exactly what this check
|
||||
/// surfaces per-row.
|
||||
async fn count_total(&self) -> Option<u64> {
|
||||
let row: Result<(i64,), sqlx::Error> = sqlx::query_as("SELECT COUNT(*) FROM storage.blobs")
|
||||
.fetch_one(self.pool.as_ref())
|
||||
.await;
|
||||
match row {
|
||||
Ok((n,)) => Some(n.max(0) as u64),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "backend_consistency.count_total_failed",
|
||||
error = %e,
|
||||
"count_total failed — run will not surface a progress bar"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn progress_kind(&self) -> ProgressKind {
|
||||
// Approximate — the denominator (DB count) is a proxy for
|
||||
// the backend count. Deviation is meaningful (see the
|
||||
// count_total doc).
|
||||
ProgressKind::Approximate
|
||||
}
|
||||
|
||||
async fn run_resumable(
|
||||
&self,
|
||||
store: &dyn JobStore,
|
||||
_args: &JobRunArgs,
|
||||
resume_cursor: Option<Vec<u8>>,
|
||||
) -> RunOutcome {
|
||||
// Cursor = opaque backend continuation token, UTF-8-encoded.
|
||||
// Each backend defines its own format (local = shard/hash,
|
||||
// S3 = ListObjectsV2 continuation token, Azure = list
|
||||
// marker); the tenant just passes it through.
|
||||
let mut cursor: Option<String> = match resume_cursor {
|
||||
None => None,
|
||||
Some(bytes) if bytes.is_empty() => None,
|
||||
Some(bytes) => match String::from_utf8(bytes) {
|
||||
Ok(s) => Some(s),
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("invalid cursor: not valid UTF-8: {e}"),
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let mut finding_count = 0u64;
|
||||
|
||||
loop {
|
||||
// Cancel poll between batches.
|
||||
match store.status().await {
|
||||
Ok(RunStatus::CancelRequested) => {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "backend_consistency.cancelled",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"backend_consistency cancelled cooperatively, pausing"
|
||||
);
|
||||
return RunOutcome::Paused {
|
||||
cursor: cursor
|
||||
.as_ref()
|
||||
.map(|s| s.as_bytes().to_vec())
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("status poll: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch next batch from the backend. `BlobListPage`
|
||||
// splits canonical blobs (checked for orphan) from
|
||||
// "unknown" entries (sidecar files, foreign namespaces —
|
||||
// emitted as informational notices).
|
||||
let page = match self
|
||||
.backend
|
||||
.list_blob_hashes(cursor.clone(), BATCH_SIZE)
|
||||
.await
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
// Backend refuses / can't enumerate. First-batch
|
||||
// failure = we emit ONE run-level anomaly and
|
||||
// complete cleanly (the run stays useful — the
|
||||
// operator learns why nothing was checked
|
||||
// instead of getting a red error). Mid-scan
|
||||
// failure = we fail the run.
|
||||
|
||||
let is_first_batch = cursor.is_none() && finding_count == 0;
|
||||
if is_first_batch {
|
||||
// No local increment — the local
|
||||
// `finding_count` is only used for the
|
||||
// completion log below, but this branch
|
||||
// returns immediately. The finding IS
|
||||
// persisted + counted in `stats.finding_count`
|
||||
// by `record_or_log` → `store.record_finding`.
|
||||
record_or_log(
|
||||
store,
|
||||
BACKEND_CONSISTENCY_JOB_NAME,
|
||||
"backend_unenumerable",
|
||||
"anomaly",
|
||||
None,
|
||||
serde_json::json!({
|
||||
"backend": self.backend.backend_type(),
|
||||
"error": format!("{e}"),
|
||||
"note": "backend refused enumeration; no per-blob orphan probes attempted",
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "backend_consistency.unenumerable",
|
||||
run_id = %store.run_id(),
|
||||
backend = self.backend.backend_type(),
|
||||
"backend refused enumeration (typical during migration or on backends without list support)"
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
return RunOutcome::Failed {
|
||||
message: format!("backend list failed mid-scan: {e}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let grace_cutoff = Utc::now() - CREATE_GRACE;
|
||||
|
||||
// Non-canonical files in the blob namespace — sidecars,
|
||||
// wrong extensions, foreign namespaces. Informational
|
||||
// only (severity `anomaly`, blue notice pill). Emitted
|
||||
// BEFORE the blob orphan probes so operators see them
|
||||
// grouped near the top of the findings list per batch.
|
||||
// Grace-window filter applies here too — a temp file
|
||||
// being written should not fire a notice.
|
||||
for unknown in &page.unknowns {
|
||||
if let Some(mtime) = unknown.mtime
|
||||
&& mtime > grace_cutoff
|
||||
{
|
||||
continue;
|
||||
}
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
BACKEND_CONSISTENCY_JOB_NAME,
|
||||
"unknown_backend_file",
|
||||
"anomaly",
|
||||
None,
|
||||
serde_json::json!({
|
||||
"path": unknown.path,
|
||||
"mtime": unknown.mtime.map(|t| t.to_rfc3339()),
|
||||
"backend": self.backend.backend_type(),
|
||||
"note": "non-canonical file in blob namespace (sidecar / wrong extension); not managed by dedup",
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
if page.blobs.is_empty() && page.next_cursor.is_none() {
|
||||
// Nothing more to enumerate. Empty-blobs batches
|
||||
// with unknowns still emitted above are fine — we
|
||||
// fall through to completion.
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "backend_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"backend_consistency completed with {} finding(s)",
|
||||
finding_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
|
||||
// Batch DB probe: which of these hashes have a
|
||||
// `storage.blobs` row? One `WHERE hash = ANY($1)` per
|
||||
// batch — indexed lookup, cheap even on millions of
|
||||
// rows.
|
||||
let batch_hashes: Vec<String> = page.blobs.iter().map(|e| e.hash.clone()).collect();
|
||||
let db_present: HashSet<String> = if batch_hashes.is_empty() {
|
||||
HashSet::new()
|
||||
} else {
|
||||
match sqlx::query_as::<_, (String,)>(
|
||||
r#"SELECT hash FROM storage.blobs WHERE hash = ANY($1)"#,
|
||||
)
|
||||
.bind(&batch_hashes[..])
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
{
|
||||
Ok(rows) => rows.into_iter().map(|(h,)| h).collect(),
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("db probe: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for entry in &page.blobs {
|
||||
if db_present.contains(&entry.hash) {
|
||||
continue;
|
||||
}
|
||||
if let Some(mtime) = entry.mtime
|
||||
&& mtime > grace_cutoff
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
BACKEND_CONSISTENCY_JOB_NAME,
|
||||
"orphan_blob",
|
||||
"inconsistent",
|
||||
None,
|
||||
serde_json::json!({
|
||||
"hash": entry.hash,
|
||||
"mtime": entry.mtime.map(|t| t.to_rfc3339()),
|
||||
"backend": self.backend.backend_type(),
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Advance cursor + checkpoint. Scanned count tracks
|
||||
// both blobs and unknowns since we walked both.
|
||||
let batch_len = (page.blobs.len() + page.unknowns.len()) as u64;
|
||||
cursor = page.next_cursor;
|
||||
let cursor_bytes = cursor
|
||||
.as_ref()
|
||||
.map(|s| s.as_bytes().to_vec())
|
||||
.unwrap_or_default();
|
||||
if let Err(e) = store.checkpoint(cursor_bytes, batch_len).await {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("checkpoint: {e}"),
|
||||
};
|
||||
}
|
||||
|
||||
// Backend returned no next_cursor → enumeration
|
||||
// complete. Emit the completion log and return.
|
||||
if cursor.is_none() {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "backend_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"backend_consistency completed with {} finding(s)",
|
||||
finding_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,494 @@
|
||||
//! Fourth tenant of Part 2 (recoverable-run engine).
|
||||
//!
|
||||
//! Iterates `storage.blobs` — the content-addressable registry —
|
||||
//! and verifies each row against the physical backend AND against
|
||||
//! the reference-counting invariants that `dedup_gc` relies on.
|
||||
//!
|
||||
//! Three per-row checks (subject-iteration principle in action —
|
||||
//! one walk, multiple branches):
|
||||
//!
|
||||
//! * `blob_missing_from_backend` (severity `data_loss`) — the DB
|
||||
//! row says the hash exists but `BlobStorageBackend::blob_exists`
|
||||
//! returns false. Bytes gone from disk / S3 / Azure. Any file
|
||||
//! whose manifest references this hash (or whose whole-file
|
||||
//! `blob_hash` points at it) will fail to read.
|
||||
//!
|
||||
//! * `blob_corrupted` (severity `data_loss`, deep mode only) —
|
||||
//! bytes exist on the backend but their BLAKE3 no longer matches
|
||||
//! the hash under which they're indexed. Silent bit-rot. Only
|
||||
//! runs when the operator passes `?deep=true` because it costs a
|
||||
//! full read of every blob.
|
||||
//!
|
||||
//! * `refcount_mismatch` (severity `inconsistent`) —
|
||||
//! `storage.blobs.ref_count` disagrees with the actual reference
|
||||
//! count computed from `storage.files.blob_hash` +
|
||||
//! `storage.chunk_manifests.chunk_hashes[]`. Under-count means
|
||||
//! dedup GC could prematurely reap a live blob; over-count means
|
||||
//! a blob is being pinned longer than needed. Content-safe either
|
||||
//! way (the storage.blobs row is fine, the counter is wrong).
|
||||
//!
|
||||
//! ### Complements `files_consistency`
|
||||
//!
|
||||
//! `files_consistency` (Slice 6/10) iterates files and verifies DB
|
||||
//! integrity. `blobs_consistency` iterates the storage registry and
|
||||
//! verifies physical existence + counter integrity. Together they
|
||||
//! cover both sides of the reference graph. Neither doubles the
|
||||
//! other's work — probing per-blob (here) instead of per-file-chunk
|
||||
//! preserves dedup savings: a chunk shared by 5 files gets probed
|
||||
//! ONCE.
|
||||
//!
|
||||
//! ### Not covered here
|
||||
//!
|
||||
//! * **Orphan bytes on the backend** (files on disk with no DB row)
|
||||
//! — belongs in the future `backend_consistency` tenant which
|
||||
//! iterates the backend itself. Requires the `list_blob_hashes`
|
||||
//! trait extension and per-backend enumeration impls.
|
||||
//! * **Manifest-level integrity** (`storage.chunk_manifests` rows
|
||||
//! pointing at reaped chunks) — already covered by
|
||||
//! `files_consistency::chunk_missing`.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use sqlx::PgPool;
|
||||
|
||||
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||
use crate::infrastructure::scheduler::{
|
||||
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome,
|
||||
RunStatus, record_or_log,
|
||||
};
|
||||
|
||||
pub const BLOBS_CONSISTENCY_JOB_NAME: &str = "blobs_consistency";
|
||||
|
||||
/// Rows per batch. Blobs are numerous (millions on a busy install)
|
||||
/// but per-row work is one indexed backend probe + one indexed SQL
|
||||
/// ref-count query. 200 balances cancel-poll cadence against
|
||||
/// round-trip amortisation.
|
||||
const BATCH_SIZE: i64 = 200;
|
||||
|
||||
/// Grace window — rows created within this window are skipped by
|
||||
/// the physical-existence probe because the write path is
|
||||
/// durability-before-visibility: `dedup_service` writes bytes, then
|
||||
/// registers the row a few ms later. A scan catching a row
|
||||
/// mid-write would false-positive it as `blob_missing_from_backend`.
|
||||
/// Same shape `dedup_gc` uses (see its `grace_secs`).
|
||||
const CREATE_GRACE: Duration = Duration::hours(1);
|
||||
|
||||
/// Cap on reverse-lookup file names surfaced in a finding's detail.
|
||||
/// Keeps detail JSON size bounded when a broken blob is referenced
|
||||
/// by hundreds of files.
|
||||
const AFFECTED_FILES_SAMPLE: i64 = 5;
|
||||
|
||||
pub struct BlobsConsistencyCheck {
|
||||
pool: Arc<PgPool>,
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
}
|
||||
|
||||
impl BlobsConsistencyCheck {
|
||||
pub fn new(pool: Arc<PgPool>, backend: Arc<dyn BlobStorageBackend>) -> Self {
|
||||
Self { pool, backend }
|
||||
}
|
||||
|
||||
pub async fn register_recoverable_job(
|
||||
self: Arc<Self>,
|
||||
registry: &JobRegistry,
|
||||
provider: &Arc<dyn JobStoreProvider>,
|
||||
) -> Arc<Self> {
|
||||
registry
|
||||
.register_recoverable_job(self.clone(), provider.clone(), None)
|
||||
.await;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
struct BlobRow {
|
||||
hash: String,
|
||||
size: i64,
|
||||
ref_count: i32,
|
||||
created_at: DateTime<Utc>,
|
||||
/// Real reference count derived from the actual references —
|
||||
/// files' whole-file `blob_hash` PLUS every chunk hash across
|
||||
/// `storage.chunk_manifests`. Compared to `ref_count` (the
|
||||
/// stored counter) to detect drift.
|
||||
actual_ref_count: i64,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RecoverableJobHandler for BlobsConsistencyCheck {
|
||||
fn name(&self) -> &str {
|
||||
BLOBS_CONSISTENCY_JOB_NAME
|
||||
}
|
||||
|
||||
/// Definitive count. `storage.blobs` PK scan is index-only;
|
||||
/// even at millions of rows it's sub-second on modern PG.
|
||||
async fn count_total(&self) -> Option<u64> {
|
||||
let row: Result<(i64,), sqlx::Error> = sqlx::query_as("SELECT COUNT(*) FROM storage.blobs")
|
||||
.fetch_one(self.pool.as_ref())
|
||||
.await;
|
||||
match row {
|
||||
Ok((n,)) => Some(n.max(0) as u64),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "blobs_consistency.count_total_failed",
|
||||
error = %e,
|
||||
"count_total failed — run will not surface a progress bar"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_resumable(
|
||||
&self,
|
||||
store: &dyn JobStore,
|
||||
args: &JobRunArgs,
|
||||
resume_cursor: Option<Vec<u8>>,
|
||||
) -> RunOutcome {
|
||||
// Cursor = the last-visited `hash` string, UTF-8-encoded. On
|
||||
// resume, we walk `WHERE hash > $cursor` in ASC order. First
|
||||
// batch: NULL cursor → start from the smallest hash.
|
||||
let mut cursor: Option<String> = match resume_cursor {
|
||||
None => None,
|
||||
Some(bytes) if bytes.is_empty() => None,
|
||||
Some(bytes) => match String::from_utf8(bytes) {
|
||||
Ok(s) => Some(s),
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("invalid cursor: not valid UTF-8: {e}"),
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// Per-run finding counter (feeds outer JobOutcome extras via
|
||||
// stats.finding_count — actual persistence happens in
|
||||
// `record_finding` on each emission).
|
||||
let mut finding_count = 0u64;
|
||||
|
||||
// Deep mode = re-hash bytes for bit-rot detection. Logged
|
||||
// once at run start so operators tailing tracing know why the
|
||||
// scan is taking hours.
|
||||
if args.deep {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "blobs_consistency.deep_mode_active",
|
||||
run_id = %store.run_id(),
|
||||
"deep mode: re-reading + re-hashing every blob (bit-rot detection)"
|
||||
);
|
||||
}
|
||||
|
||||
loop {
|
||||
// Cooperative cancel poll between batches.
|
||||
match store.status().await {
|
||||
Ok(RunStatus::CancelRequested) => {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "blobs_consistency.cancelled",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"blobs_consistency cancelled cooperatively, pausing"
|
||||
);
|
||||
return RunOutcome::Paused {
|
||||
cursor: cursor
|
||||
.as_ref()
|
||||
.map(|s| s.as_bytes().to_vec())
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("status poll: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch the next batch. Per-row `actual_ref_count`
|
||||
// computed inline via correlated subqueries — one for
|
||||
// legacy whole-file references (`files.blob_hash`), one
|
||||
// for CDC chunk references (`chunk_manifests.chunk_hashes`).
|
||||
// GIN index on `chunk_hashes` (migration
|
||||
// 20260628000000_delta_upload_gin_index) makes the
|
||||
// `= ANY(chunk_hashes)` probe cheap.
|
||||
// `storage.blobs.ref_count` semantics — what the invariant
|
||||
// dedup_service maintains actually is:
|
||||
//
|
||||
// ref_count = (number of chunk_manifests whose
|
||||
// chunk_hashes[] contains this hash)
|
||||
// + (number of files.blob_hash pointing at
|
||||
// this hash on the LEGACY whole-file path
|
||||
// — i.e. files with NO manifest for their
|
||||
// blob_hash)
|
||||
//
|
||||
// Naively `COUNT(files) + COUNT(manifests referring)`
|
||||
// double-counts single-chunk CDC files: for a file whose
|
||||
// whole-file hash == its single chunk's hash (any file
|
||||
// small enough to fit in one CDC chunk — under ~256 KB
|
||||
// average), the file appears BOTH in `files.blob_hash`
|
||||
// AND in the manifest's `chunk_hashes[]`. The `NOT
|
||||
// EXISTS` clause below excludes CDC-path files from the
|
||||
// legacy count so the two terms don't overlap.
|
||||
let rows: Vec<BlobRow> = match sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
b.hash AS hash,
|
||||
b.size AS size,
|
||||
b.ref_count AS ref_count,
|
||||
b.created_at AS created_at,
|
||||
(
|
||||
(SELECT COUNT(*) FROM storage.files f
|
||||
WHERE f.blob_hash = b.hash
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM storage.chunk_manifests m
|
||||
WHERE m.file_hash = f.blob_hash
|
||||
))
|
||||
+ (SELECT COUNT(*) FROM storage.chunk_manifests m
|
||||
WHERE b.hash = ANY(m.chunk_hashes))
|
||||
)::bigint AS actual_ref_count
|
||||
FROM storage.blobs b
|
||||
WHERE ($1::text IS NULL OR b.hash > $1)
|
||||
ORDER BY b.hash
|
||||
LIMIT $2
|
||||
"#,
|
||||
)
|
||||
.bind(cursor.as_deref())
|
||||
.bind(BATCH_SIZE)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("batch fetch: {e}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if rows.is_empty() {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "blobs_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
deep = args.deep,
|
||||
"blobs_consistency completed with {} finding(s)",
|
||||
finding_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
|
||||
let grace_cutoff = Utc::now() - CREATE_GRACE;
|
||||
|
||||
for row in &rows {
|
||||
// (1) refcount_mismatch — content-safe check, cheap,
|
||||
// always runs. Emitted BEFORE the physical probe so
|
||||
// a broken-and-miscounted blob shows both findings.
|
||||
if row.ref_count as i64 != row.actual_ref_count {
|
||||
finding_count += 1;
|
||||
let affected = affected_files(self.pool.as_ref(), &row.hash).await;
|
||||
record_or_log(
|
||||
store,
|
||||
BLOBS_CONSISTENCY_JOB_NAME,
|
||||
"refcount_mismatch",
|
||||
"inconsistent",
|
||||
None, // hash isn't a UUID; resource identifier lives in detail
|
||||
serde_json::json!({
|
||||
"hash": row.hash,
|
||||
"stored": row.ref_count,
|
||||
"actual": row.actual_ref_count,
|
||||
"delta": row.actual_ref_count - row.ref_count as i64,
|
||||
"size": row.size,
|
||||
"affected_files": affected,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Skip physical probes for rows within the write
|
||||
// grace window — writes-in-flight would false-positive.
|
||||
if row.created_at > grace_cutoff {
|
||||
continue;
|
||||
}
|
||||
|
||||
// (2) blob_missing_from_backend — normal mode
|
||||
// physical existence probe. Fails-open on backend
|
||||
// error (log + skip): a transient S3 network blip
|
||||
// shouldn't produce a flood of false data_loss
|
||||
// findings.
|
||||
let exists = match self.backend.blob_exists(&row.hash).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "blobs_consistency.blob_exists_error",
|
||||
run_id = %store.run_id(),
|
||||
hash = %row.hash,
|
||||
error = %e,
|
||||
"blob_exists probe failed; skipping this row"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !exists {
|
||||
finding_count += 1;
|
||||
let affected = affected_files(self.pool.as_ref(), &row.hash).await;
|
||||
record_or_log(
|
||||
store,
|
||||
BLOBS_CONSISTENCY_JOB_NAME,
|
||||
"blob_missing_from_backend",
|
||||
"data_loss",
|
||||
None,
|
||||
serde_json::json!({
|
||||
"hash": row.hash,
|
||||
"size": row.size,
|
||||
"ref_count": row.ref_count,
|
||||
"affected_files": affected,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
// No point re-hashing bytes that aren't there.
|
||||
continue;
|
||||
}
|
||||
|
||||
// (3) blob_corrupted — DEEP MODE only. Read the
|
||||
// whole blob, recompute BLAKE3, compare to the hash
|
||||
// it's indexed under. Any mismatch = silent bit-rot.
|
||||
//
|
||||
// Finding fields:
|
||||
// * `hash` — expected hash (the key the blob is
|
||||
// indexed under in `storage.blobs`).
|
||||
// * `computed_hash` — what BLAKE3 of the current
|
||||
// bytes actually produces. Diagnostic: a
|
||||
// one-bit flip vs a truncation vs a whole-file
|
||||
// swap all leave distinctive signatures.
|
||||
// `expected_hash` was NOT reused as a name to
|
||||
// avoid mistaking it for "the hash we expect to
|
||||
// see on disk (i.e. what will fix this)".
|
||||
if args.deep {
|
||||
match recompute_hash(self.backend.as_ref(), &row.hash).await {
|
||||
Ok(computed_hash) if computed_hash == row.hash => {}
|
||||
Ok(computed_hash) => {
|
||||
finding_count += 1;
|
||||
let affected = affected_files(self.pool.as_ref(), &row.hash).await;
|
||||
record_or_log(
|
||||
store,
|
||||
BLOBS_CONSISTENCY_JOB_NAME,
|
||||
"blob_corrupted",
|
||||
"data_loss",
|
||||
None,
|
||||
serde_json::json!({
|
||||
"hash": row.hash,
|
||||
"computed_hash": computed_hash,
|
||||
"size": row.size,
|
||||
"ref_count": row.ref_count,
|
||||
"affected_files": affected,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "blobs_consistency.recompute_hash_error",
|
||||
run_id = %store.run_id(),
|
||||
hash = %row.hash,
|
||||
error = %e,
|
||||
"recompute_hash failed; not a corruption signal on its own"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Advance cursor + checkpoint.
|
||||
let last_hash = rows.last().map(|r| r.hash.clone()).expect("non-empty rows");
|
||||
cursor = Some(last_hash.clone());
|
||||
let batch_len = rows.len() as u64;
|
||||
if let Err(e) = store.checkpoint(last_hash.into_bytes(), batch_len).await {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("checkpoint: {e}"),
|
||||
};
|
||||
}
|
||||
|
||||
if (rows.len() as i64) < BATCH_SIZE {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "blobs_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
deep = args.deep,
|
||||
"blobs_consistency completed with {} finding(s)",
|
||||
finding_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sample of file names that reference this blob — either directly
|
||||
/// (`files.blob_hash = $hash`, legacy pre-CDC) or transitively via a
|
||||
/// manifest (`chunk_hashes @> ARRAY[$hash]`, post-CDC dominant path).
|
||||
/// Capped so a chunk shared by 10 000 files doesn't blow up the
|
||||
/// finding detail JSON. Order is arbitrary — sampling for
|
||||
/// diagnosis, not enumeration.
|
||||
async fn affected_files(pool: &PgPool, hash: &str) -> Vec<String> {
|
||||
let rows: Vec<(String,)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT DISTINCT f.name
|
||||
FROM storage.files f
|
||||
WHERE f.blob_hash = $1
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM storage.chunk_manifests m
|
||||
WHERE m.file_hash = f.blob_hash
|
||||
AND $1 = ANY(m.chunk_hashes)
|
||||
)
|
||||
LIMIT $2
|
||||
"#,
|
||||
)
|
||||
.bind(hash)
|
||||
.bind(AFFECTED_FILES_SAMPLE)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
rows.into_iter().map(|(n,)| n).collect()
|
||||
}
|
||||
|
||||
/// Deep-mode helper — read the blob from the backend and recompute
|
||||
/// its BLAKE3 hash. Returns `Ok(true)` when the recomputed hash
|
||||
/// matches `expected_hash` (byte for byte), `Ok(false)` on mismatch
|
||||
/// (bit-rot), `Err(_)` on any backend-side error (network blip,
|
||||
/// permission issue) — callers log-and-skip errors since a transient
|
||||
/// failure isn't a corruption signal.
|
||||
/// Deep-mode helper — read the blob from the backend and recompute
|
||||
/// its BLAKE3 hash. Returns the recomputed hex string; callers
|
||||
/// compare against the expected hash themselves. Returning the
|
||||
/// actual hash (not just a bool) lets the finding surface WHAT the
|
||||
/// bytes now hash to, which is diagnostic gold: a specific one-bit
|
||||
/// flip has a very different signature from a chunk-boundary
|
||||
/// corruption or a truncated read. `Err(_)` on backend-side error
|
||||
/// (network blip, permission issue) — callers log-and-skip since
|
||||
/// transient failure isn't a corruption signal.
|
||||
async fn recompute_hash(
|
||||
backend: &dyn BlobStorageBackend,
|
||||
expected_hash: &str,
|
||||
) -> Result<String, crate::common::errors::DomainError> {
|
||||
use crate::common::errors::DomainError;
|
||||
use futures::StreamExt;
|
||||
|
||||
let mut stream = backend.get_blob_stream(expected_hash).await?;
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let bytes = chunk.map_err(|e| {
|
||||
DomainError::internal_error("BlobsConsistency", format!("stream read: {e}"))
|
||||
})?;
|
||||
hasher.update(&bytes);
|
||||
}
|
||||
|
||||
Ok(hasher.finalize().to_hex().to_string())
|
||||
}
|
||||
@@ -412,6 +412,29 @@ impl BlobStorageBackend for CachedBlobBackend {
|
||||
let path = self.cached_path(hash);
|
||||
if path.exists() { Some(path) } else { None }
|
||||
}
|
||||
|
||||
/// Enumeration MUST delegate to the primary (inner) backend, not
|
||||
/// the local cache. The cache is by definition a subset (only
|
||||
/// recently-accessed blobs); walking the cache would look like
|
||||
/// "most of my blobs are orphans" from the tenant's perspective.
|
||||
/// The inner backend is the authoritative "what exists" source.
|
||||
fn list_blob_hashes(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: usize,
|
||||
) -> Pin<
|
||||
Box<
|
||||
dyn std::future::Future<
|
||||
Output = Result<
|
||||
crate::application::ports::blob_storage_ports::BlobListPage,
|
||||
DomainError,
|
||||
>,
|
||||
> + Send
|
||||
+ '_,
|
||||
>,
|
||||
> {
|
||||
self.inner.list_blob_hashes(cursor, limit)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Cache internals (miss path + population) ───────────────────────
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
//! "Run all consistency checks" coordinator.
|
||||
//!
|
||||
//! A plain [`JobHandler`] (not [`RecoverableJobHandler`]) — it walks
|
||||
//! nothing, holds no cursor. Its whole job is to snapshot the
|
||||
//! registry, filter to names ending `_consistency`, and dispatch each
|
||||
//! sequentially via `registry.trigger(name, args)`. Sub-jobs receive
|
||||
//! the SAME `JobRunArgs` the batch was invoked with — so
|
||||
//! `?deep=true` on the batch propagates to whichever tenants respect
|
||||
//! it (currently future `storage_consistency`, but the plumbing is in
|
||||
//! place).
|
||||
//!
|
||||
//! ### Why a wrapper instead of a bulk endpoint
|
||||
//!
|
||||
//! Operators want one click for "run all". Building a general-purpose
|
||||
//! `POST /api/admin/jobs/*/trigger` group-endpoint would need its own
|
||||
//! auth path, its own concurrency envelope, its own outcome shape.
|
||||
//! A wrapper JobHandler reuses ALL of that infrastructure:
|
||||
//!
|
||||
//! - Same admin URL: `POST /api/admin/jobs/consistency_batch/trigger`.
|
||||
//! - Same audit trail: one line per batch invocation.
|
||||
//! - Same exclusivity primitive: the Part 1 per-job semaphore keeps
|
||||
//! two `consistency_batch` runs from stomping each other. Two
|
||||
//! batches (say, one `?deep=false` + one `?deep=true`) share the
|
||||
//! same lock — an admin cannot accidentally start a deep pass
|
||||
//! while a normal one is still walking.
|
||||
//! - Same JSON outcome envelope — `per_check` lands under
|
||||
//! `outcome.extra`, which the admin UI can drill into without
|
||||
//! inventing a new response schema.
|
||||
//!
|
||||
//! ### Why the batch always returns `Ok`
|
||||
//!
|
||||
//! The batch's job is **dispatch**, not investigation. A child
|
||||
//! failing means the child failed — not the batch. Failures surface
|
||||
//! in `extra.per_check[<name>].outcome = "err"`; the operator drills
|
||||
//! in. Reporting the batch itself as `Err` would confuse the metric
|
||||
//! "did the batch run" with "did all children succeed", which are
|
||||
//! genuinely different questions.
|
||||
//!
|
||||
//! ### Registration ordering
|
||||
//!
|
||||
//! `consistency_batch` MUST register AFTER every tenant it dispatches
|
||||
//! — but only for a debug-affordance reason: the ordering of the
|
||||
//! `GET /api/admin/jobs` response mirrors registration order, and
|
||||
//! having the wrapper sit at the end of the consistency block reads
|
||||
//! more naturally. Snapshot filtering happens at RUN time, so a
|
||||
//! reversed order would still work; DI's ordering is aesthetic.
|
||||
//!
|
||||
//! ### Arc cycle avoidance
|
||||
//!
|
||||
//! [`ConsistencyBatch`] holds a `Weak<JobRegistry>` — the registry
|
||||
//! owns an `Arc<dyn JobHandler>` for the batch, and the batch needs
|
||||
//! access back to `trigger`. A strong `Arc<JobRegistry>` inside the
|
||||
//! handler would leak the registry forever. Upgrading the weak on
|
||||
//! each `run()` is cheap (one refcount bump) and gracefully surfaces
|
||||
//! "registry dropped mid-shutdown" as an error rather than a hang.
|
||||
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs};
|
||||
|
||||
pub const CONSISTENCY_BATCH_JOB_NAME: &str = "consistency_batch";
|
||||
|
||||
pub struct ConsistencyBatch {
|
||||
registry: Weak<JobRegistry>,
|
||||
}
|
||||
|
||||
impl ConsistencyBatch {
|
||||
pub fn new(registry: &Arc<JobRegistry>) -> Self {
|
||||
Self {
|
||||
registry: Arc::downgrade(registry),
|
||||
}
|
||||
}
|
||||
|
||||
/// Chainable self-registration — mirrors the per-tenant helpers.
|
||||
/// On-demand only; there is no periodic tick (operators fire it
|
||||
/// when they want to sweep, or the frontend "run all" button in
|
||||
/// `/admin/jobs` triggers it once the UI ships).
|
||||
pub async fn register_job(self: Arc<Self>, registry: &JobRegistry) -> Arc<Self> {
|
||||
registry.register(self.clone(), None, None).await;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for ConsistencyBatch {
|
||||
fn name(&self) -> &str {
|
||||
CONSISTENCY_BATCH_JOB_NAME
|
||||
}
|
||||
|
||||
async fn run(&self, args: &JobRunArgs) -> JobOutcome {
|
||||
// Upgrade the Weak. Only fails if the registry has been
|
||||
// dropped — which can only happen during process shutdown,
|
||||
// in which case the scheduler is winding down anyway.
|
||||
let registry = match self.registry.upgrade() {
|
||||
Some(r) => r,
|
||||
None => {
|
||||
return JobOutcome::err(
|
||||
"consistency_batch: registry dropped (shutdown in progress?)",
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Snapshot + filter. `snapshot_all` would give us Arc<JobEntry>
|
||||
// handles too, but we don't need them — `registry.trigger`
|
||||
// does the lookup by name itself. `snapshot` returns the
|
||||
// per-job public DTOs, which is exactly the shape we want.
|
||||
let targets: Vec<String> = registry
|
||||
.snapshot()
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|s| s.name.ends_with("_consistency") && s.name != CONSISTENCY_BATCH_JOB_NAME)
|
||||
.map(|s| s.name)
|
||||
.collect();
|
||||
|
||||
let mut per_check = serde_json::Map::new();
|
||||
let mut ok_count = 0u64;
|
||||
let mut err_count = 0u64;
|
||||
|
||||
// Sequential dispatch. Parallel would give us tail-latency
|
||||
// wins but also multiplies DB pressure — the maintenance pool
|
||||
// is shared with the periodic sweeps that keep running while
|
||||
// the batch runs. Sequential keeps memory + IO envelope
|
||||
// predictable; the batch is a "run once in a while, take as
|
||||
// long as it takes" workflow, not a hot path.
|
||||
for name in &targets {
|
||||
let child = registry.trigger(name, args).await;
|
||||
match &child {
|
||||
Some(JobOutcome::Ok { count, extra }) => {
|
||||
ok_count += 1;
|
||||
let mut entry = json!({
|
||||
"outcome": "ok",
|
||||
"count": count,
|
||||
});
|
||||
if !extra.is_null() {
|
||||
// Preserve per-check `extra` (e.g.
|
||||
// drives_consistency emits drift counts here
|
||||
// once `run_findings` lands). Nested under
|
||||
// its own key so operators reading
|
||||
// `per_check[name]` see a stable shape.
|
||||
entry["extra"] = extra.clone();
|
||||
}
|
||||
per_check.insert(name.clone(), entry);
|
||||
}
|
||||
Some(JobOutcome::Err { message }) => {
|
||||
err_count += 1;
|
||||
per_check.insert(
|
||||
name.clone(),
|
||||
json!({
|
||||
"outcome": "err",
|
||||
"message": message,
|
||||
}),
|
||||
);
|
||||
}
|
||||
None => {
|
||||
// Race: the job disappeared between snapshot and
|
||||
// trigger. In practice this only happens if some
|
||||
// future code path deregisters a tenant at
|
||||
// runtime. Report so operators see it in the
|
||||
// batch outcome and can chase the cause.
|
||||
err_count += 1;
|
||||
per_check.insert(
|
||||
name.clone(),
|
||||
json!({
|
||||
"outcome": "err",
|
||||
"message": "job no longer registered (race with deregistration)",
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
JobOutcome::ok_with(
|
||||
targets.len() as u64,
|
||||
json!({
|
||||
"per_check": per_check,
|
||||
"deep": args.deep,
|
||||
"force": args.force,
|
||||
"ok": ok_count,
|
||||
"err": err_count,
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -3127,6 +3127,29 @@ impl DedupPort for DedupService {
|
||||
/// and admin URLs (`POST /api/admin/jobs/dedup_gc/trigger`).
|
||||
pub const DEDUP_GC_JOB_NAME: &str = "dedup_gc";
|
||||
|
||||
impl DedupService {
|
||||
/// Register self with the periodic-job scheduler and return the
|
||||
/// same `Arc<Self>` for DI-style chaining. **On-demand only** —
|
||||
/// registered with `interval = None`. The periodic GC role
|
||||
/// belongs to trash cleanup (which invokes `garbage_collect()`
|
||||
/// inline as its tail step); a duplicate scheduled tick here
|
||||
/// would double the reclamation work. Registration exists solely
|
||||
/// to expose the admin trigger uniformly.
|
||||
pub async fn register(
|
||||
self: std::sync::Arc<Self>,
|
||||
registry: &crate::infrastructure::scheduler::JobRegistry,
|
||||
) -> std::sync::Arc<Self> {
|
||||
registry
|
||||
.register(
|
||||
self.clone() as std::sync::Arc<dyn crate::infrastructure::scheduler::JobHandler>,
|
||||
None, // on-demand
|
||||
None, // no timeout
|
||||
)
|
||||
.await;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::infrastructure::scheduler::JobHandler for DedupService {
|
||||
fn name(&self) -> &str {
|
||||
|
||||
@@ -0,0 +1,689 @@
|
||||
//! First tenant of Part 2 (recoverable-run engine).
|
||||
//!
|
||||
//! Iterates `storage.drives` and reports each drive whose cached
|
||||
//! `used_bytes` differs from `SUM(files.size) WHERE NOT is_trashed`
|
||||
//! for that drive. **Read-only** — reports drift as findings but does
|
||||
//! NOT fix it. The existing `storage_reconcile` job (Part 1) is what
|
||||
//! corrects the counter; this check surfaces WHEN drift happens so
|
||||
//! operators can trace it back to root cause (missed delta call,
|
||||
//! delta failed silently, race, etc.).
|
||||
//!
|
||||
//! One check today — `used_bytes` drift — but structured so more
|
||||
//! checks can slot in as per-row branches (quota-vs-usage inversion,
|
||||
//! `kind` vs `default_for_user` invariants, ...). See memory note
|
||||
//! `project_consistency_jobs_landscape`.
|
||||
//!
|
||||
//! Findings are LOGGED to `target: "oxicloud::consistency"` for now.
|
||||
//! Persistence to `jobs.run_findings` lands with the findings-table
|
||||
//! migration (deferred; see the plan doc). Once landed, this handler
|
||||
//! swaps its `tracing::warn!` finding calls for
|
||||
//! `store.record_finding(...)` — nothing else changes.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::infrastructure::scheduler::{
|
||||
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome,
|
||||
RunStatus, record_or_log,
|
||||
};
|
||||
|
||||
pub const DRIVES_CONSISTENCY_JOB_NAME: &str = "drives_consistency";
|
||||
|
||||
/// Rows per batch. Drives are few (dozens per install), so this only
|
||||
/// matters for the cancel-poll cadence — smaller batch = more frequent
|
||||
/// status polls but more DB round-trips. 100 is comfortably fast for
|
||||
/// any realistic drive count.
|
||||
const BATCH_SIZE: i64 = 100;
|
||||
|
||||
pub struct DrivesConsistencyCheck {
|
||||
pool: Arc<PgPool>,
|
||||
}
|
||||
|
||||
impl DrivesConsistencyCheck {
|
||||
pub fn new(pool: Arc<PgPool>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Register self with the periodic-job scheduler as a recoverable
|
||||
/// job, on-demand only (no periodic tick). Follows the same
|
||||
/// chainable pattern as Part 1 tenants' `register_job` — DI stays
|
||||
/// one line.
|
||||
pub async fn register_recoverable_job(
|
||||
self: Arc<Self>,
|
||||
registry: &JobRegistry,
|
||||
provider: &Arc<dyn JobStoreProvider>,
|
||||
) -> Arc<Self> {
|
||||
registry
|
||||
.register_recoverable_job(self.clone(), provider.clone(), None)
|
||||
.await;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RecoverableJobHandler for DrivesConsistencyCheck {
|
||||
fn name(&self) -> &str {
|
||||
DRIVES_CONSISTENCY_JOB_NAME
|
||||
}
|
||||
|
||||
/// Definitive count — one row per drive, table is tiny (dozens per
|
||||
/// install), COUNT(*) is trivially fast. Enables progress bar on
|
||||
/// the admin UI.
|
||||
async fn count_total(&self) -> Option<u64> {
|
||||
let row: Result<(i64,), sqlx::Error> =
|
||||
sqlx::query_as("SELECT COUNT(*) FROM storage.drives")
|
||||
.fetch_one(self.pool.as_ref())
|
||||
.await;
|
||||
match row {
|
||||
Ok((n,)) => Some(n.max(0) as u64),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "drives_consistency.count_total_failed",
|
||||
error = %e,
|
||||
"count_total failed — run will not surface a progress bar"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_resumable(
|
||||
&self,
|
||||
store: &dyn JobStore,
|
||||
_args: &JobRunArgs,
|
||||
resume_cursor: Option<Vec<u8>>,
|
||||
) -> RunOutcome {
|
||||
// Decode cursor. Convention for this job: 16 raw UUID bytes,
|
||||
// or empty/absent = start from the beginning.
|
||||
let mut cursor: Option<Uuid> = match resume_cursor {
|
||||
None => None,
|
||||
Some(bytes) if bytes.is_empty() => None,
|
||||
Some(bytes) if bytes.len() == 16 => {
|
||||
let mut arr = [0u8; 16];
|
||||
arr.copy_from_slice(&bytes);
|
||||
Some(Uuid::from_bytes(arr))
|
||||
}
|
||||
Some(bytes) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("invalid cursor: expected 16 bytes, got {}", bytes.len()),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let mut drift_count = 0u64;
|
||||
|
||||
loop {
|
||||
// Cancel poll BETWEEN batches — the cooperative cancel
|
||||
// contract (`RecoverableJobHandler` trait doc).
|
||||
match store.status().await {
|
||||
Ok(RunStatus::CancelRequested) => {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "drives_consistency.cancelled",
|
||||
run_id = %store.run_id(),
|
||||
drift_count = drift_count,
|
||||
"drives_consistency cancelled cooperatively, pausing"
|
||||
);
|
||||
return RunOutcome::Paused {
|
||||
cursor: cursor.map(|u| u.as_bytes().to_vec()).unwrap_or_default(),
|
||||
};
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("status poll: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch next batch of drives + their actual SUM in one
|
||||
// query. LEFT JOIN via correlated subquery gets us both
|
||||
// sides in one round-trip; the storage_reconcile sweep
|
||||
// uses the same shape.
|
||||
// Grace window: skip drives created within the last hour.
|
||||
// A drive being created RIGHT NOW may still have its first
|
||||
// upload's `used_bytes` counter not-yet-incremented while
|
||||
// the `files` row is already visible — that would false-
|
||||
// positive as `stale_used_bytes`. 1h matches the window
|
||||
// `blobs_consistency` uses; same rationale (writes-in-flight).
|
||||
// NOTE: `storage.drives` has no `name` column. The drive's
|
||||
// display name lives on its root folder (see the schema
|
||||
// comment on `drives.root_folder_id` — "The display name
|
||||
// lives here"). LEFT JOIN storage.folders ON id =
|
||||
// drive.root_folder_id and read `folders.name` as the
|
||||
// drive's human identifier. `COALESCE` handles the
|
||||
// (bug-only) case where root_folder_id is NULL.
|
||||
let rows: Vec<(Uuid, String, i64, i64)> = match sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
d.id AS id,
|
||||
COALESCE(rf.name, '?') AS name,
|
||||
d.used_bytes AS used_bytes,
|
||||
COALESCE((
|
||||
SELECT SUM(size)::bigint
|
||||
FROM storage.files
|
||||
WHERE drive_id = d.id
|
||||
AND NOT is_trashed
|
||||
), 0) AS actual_bytes
|
||||
FROM storage.drives d
|
||||
LEFT JOIN storage.folders rf ON rf.id = d.root_folder_id
|
||||
WHERE ($1::uuid IS NULL OR d.id > $1)
|
||||
AND d.created_at < NOW() - INTERVAL '1 hour'
|
||||
ORDER BY d.id
|
||||
LIMIT $2
|
||||
"#,
|
||||
)
|
||||
.bind(cursor)
|
||||
.bind(BATCH_SIZE)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("batch fetch: {e}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if rows.is_empty() {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "drives_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
drift_count = drift_count,
|
||||
"drives_consistency completed with {} drift finding(s)",
|
||||
drift_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
|
||||
// Per-row check: cached vs actual. This is the ONE check
|
||||
// in v1 — more per-row branches (quota inversion, kind vs
|
||||
// default_for_user, …) slot in here.
|
||||
for (drive_id, drive_name, cached, actual) in &rows {
|
||||
if *cached != *actual {
|
||||
drift_count += 1;
|
||||
// Persisted finding via the shared helper.
|
||||
// `stale_used_bytes` + severity `inconsistent`
|
||||
// (counters wrong, content intact — the
|
||||
// reconciliation sweep will fix on its next tick).
|
||||
record_or_log(
|
||||
store,
|
||||
DRIVES_CONSISTENCY_JOB_NAME,
|
||||
"stale_used_bytes",
|
||||
"inconsistent",
|
||||
Some(*drive_id),
|
||||
serde_json::json!({
|
||||
"name": drive_name,
|
||||
"cached": cached,
|
||||
"actual": actual,
|
||||
"delta": cached - actual,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
// Advance cursor to the last row's id + checkpoint.
|
||||
let last_id = rows
|
||||
.last()
|
||||
.map(|(id, _, _, _)| *id)
|
||||
.expect("non-empty rows");
|
||||
cursor = Some(last_id);
|
||||
let batch_len = rows.len() as u64;
|
||||
if let Err(e) = store
|
||||
.checkpoint(last_id.as_bytes().to_vec(), batch_len)
|
||||
.await
|
||||
{
|
||||
return RunOutcome::Failed {
|
||||
message: format!("checkpoint: {e}"),
|
||||
};
|
||||
}
|
||||
|
||||
// Short batch = drained the drives table.
|
||||
if (rows.len() as i64) < BATCH_SIZE {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "drives_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
drift_count = drift_count,
|
||||
"drives_consistency completed with {} drift finding(s)",
|
||||
drift_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Integration tests — real PG round-trip ─────────────────────────────────
|
||||
//
|
||||
// Gated on `--cfg integration_tests` (see `just test-integration`).
|
||||
// Requires a running test PG on 5433 with `oxicloud_test` DB, schema
|
||||
// applied via `tests/common/init-test-schema.sh`. Runs:
|
||||
// just test-integration -- drives_consistency_service
|
||||
//
|
||||
// Tests exercise the full recoverable-run engine against real PG:
|
||||
// - PgJobStoreProvider::open_or_start creates a run row.
|
||||
// - Handler walks a seeded drive, checkpoints, marks Completed.
|
||||
// - `stats.scanned_count` bumped, drive row untouched (read-only).
|
||||
// - Drift is DETECTED — surfaced as a `consistency_finding` event
|
||||
// on the `oxicloud::consistency` tracing target. Captured via a
|
||||
// scoped subscriber.
|
||||
|
||||
#[cfg(integration_tests)]
|
||||
#[allow(dead_code, unused_imports)] // items are exercised by #[tokio::test]
|
||||
// fns; cargo check --lib doesn't see the
|
||||
// test entry-point call graph.
|
||||
mod integration_tests {
|
||||
use super::*;
|
||||
use crate::infrastructure::scheduler::{JobStoreProvider, OpenedRun, RunStatus};
|
||||
use sqlx::Row;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
|
||||
async fn test_pool() -> Arc<sqlx::PgPool> {
|
||||
let url = crate::integration_test_support::test_db_url();
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(4)
|
||||
.connect(&url)
|
||||
.await
|
||||
.expect("connect to test DB — run tests/common/spawn-db.sh first");
|
||||
Arc::new(pool)
|
||||
}
|
||||
|
||||
/// Seed a personal drive with `used_bytes = cached` and one file
|
||||
/// of size `actual` (post-D7 schema — no `user_id` on files /
|
||||
/// folders, drives created via the circular-FK dance).
|
||||
///
|
||||
/// `default_for_user = NULL` on the drive so we don't collide
|
||||
/// with the seeded user's real default (partial unique index).
|
||||
/// Returns the drive id.
|
||||
async fn seed_drift(pool: &sqlx::PgPool, cached: i64, actual: i64) -> Uuid {
|
||||
let owner_id: Uuid = sqlx::query("SELECT id FROM auth.users LIMIT 1")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("test DB must have at least one user")
|
||||
.get(0);
|
||||
|
||||
// Steps 1-3 MUST run inside one transaction because
|
||||
// `trg_no_orphan_root_folder` is DEFERRABLE INITIALLY DEFERRED
|
||||
// (fires at COMMIT). Autocommit-per-statement would trip the
|
||||
// trigger on the folder INSERT before the drive UPDATE gets a
|
||||
// chance to close the FK. Mirrors `DrivePgRepository::
|
||||
// create_personal_drive_atomic`.
|
||||
let mut tx = pool.begin().await.expect("begin drive-create tx");
|
||||
|
||||
// 1. Drive with kind=personal, no default_for_user (avoids
|
||||
// partial-unique conflict with owner's real default),
|
||||
// used_bytes=0 for now — we set the fake value LAST.
|
||||
let drive_id: Uuid = sqlx::query_scalar(
|
||||
r#"
|
||||
INSERT INTO storage.drives
|
||||
(kind, default_for_user, quota_bytes, used_bytes)
|
||||
VALUES ('personal', NULL, NULL, 0)
|
||||
RETURNING id
|
||||
"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.expect("insert test drive");
|
||||
|
||||
// 2. Root folder for the drive (parent_id = NULL = drive root).
|
||||
// Post-D7: only `name`, `parent_id`, `drive_id`, `created_by`,
|
||||
// `updated_by` on the INSERT — `user_id`/`path`/`ltree` are
|
||||
// dropped or derived.
|
||||
let root_folder_id: Uuid = sqlx::query_scalar(
|
||||
r#"
|
||||
INSERT INTO storage.folders
|
||||
(name, parent_id, drive_id, created_by, updated_by)
|
||||
VALUES ('drift-test-root', NULL, $1, $2, $2)
|
||||
RETURNING id
|
||||
"#,
|
||||
)
|
||||
.bind(drive_id)
|
||||
.bind(owner_id)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.expect("insert root folder for test drive");
|
||||
|
||||
// 3. Close the circular FK: drive.root_folder_id points at
|
||||
// the folder we just created.
|
||||
sqlx::query("UPDATE storage.drives SET root_folder_id = $1 WHERE id = $2")
|
||||
.bind(root_folder_id)
|
||||
.bind(drive_id)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.expect("wire drive.root_folder_id");
|
||||
|
||||
tx.commit()
|
||||
.await
|
||||
.expect("commit drive-create tx (deferred trigger fires here)");
|
||||
|
||||
// 4. Optionally insert a file summing to `actual`. Also insert
|
||||
// a matching `storage.blobs` row so `trg_files_decrement_blob_ref`
|
||||
// stays happy on cleanup. Fake hash is 64-char hex derived
|
||||
// from a UUID — plausible shape, unique per fixture invocation.
|
||||
if actual > 0 {
|
||||
let fake_hash = format!(
|
||||
"{:032x}{:032x}",
|
||||
Uuid::new_v4().as_u128(),
|
||||
Uuid::new_v4().as_u128()
|
||||
);
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO storage.blobs (hash, size, ref_count, content_type)
|
||||
VALUES ($1, $2, 1, 'application/octet-stream')
|
||||
ON CONFLICT (hash) DO NOTHING
|
||||
"#,
|
||||
)
|
||||
.bind(&fake_hash)
|
||||
.bind(actual)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("insert fixture blob");
|
||||
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO storage.files
|
||||
(name, folder_id, drive_id, blob_hash, size,
|
||||
mime_type, is_trashed, created_by, updated_by)
|
||||
VALUES ($1, $2, $3, $4, $5,
|
||||
'application/octet-stream', false, $6, $6)
|
||||
"#,
|
||||
)
|
||||
.bind(format!("drift-fixture-{}.bin", Uuid::new_v4()))
|
||||
.bind(root_folder_id)
|
||||
.bind(drive_id)
|
||||
.bind(&fake_hash)
|
||||
.bind(actual)
|
||||
.bind(owner_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("insert fixture file");
|
||||
}
|
||||
|
||||
// 5. Set the artificially-wrong cached used_bytes. LAST, so
|
||||
// no INSERT-side trigger overwrites our fake (there is no
|
||||
// such trigger today, but ordering is cheap insurance).
|
||||
// Also backdate `created_at` past the tenant's 1-hour
|
||||
// grace window (`created_at < NOW() - INTERVAL '1 hour'`
|
||||
// in the SQL) — freshly-seeded fixtures are by definition
|
||||
// younger than that window and would otherwise be
|
||||
// silently skipped by the scan, leaving `scanned_count`
|
||||
// at 0 and the drift-detection assertions with nothing
|
||||
// to compare against.
|
||||
sqlx::query(
|
||||
"UPDATE storage.drives \
|
||||
SET used_bytes = $1, \
|
||||
created_at = NOW() - INTERVAL '2 hours' \
|
||||
WHERE id = $2",
|
||||
)
|
||||
.bind(cached)
|
||||
.bind(drive_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("set fake used_bytes + backdate");
|
||||
|
||||
drive_id
|
||||
}
|
||||
|
||||
async fn cleanup_test_drive(pool: &sqlx::PgPool, drive_id: Uuid) {
|
||||
// Cascading FK on files.drive_id, folders.drive_id kicks in.
|
||||
sqlx::query("DELETE FROM storage.files WHERE drive_id = $1")
|
||||
.bind(drive_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.ok();
|
||||
sqlx::query("DELETE FROM storage.folders WHERE drive_id = $1")
|
||||
.bind(drive_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.ok();
|
||||
sqlx::query("DELETE FROM storage.drives WHERE id = $1")
|
||||
.bind(drive_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.ok();
|
||||
}
|
||||
|
||||
async fn cleanup_run(pool: &sqlx::PgPool, run_id: Uuid) {
|
||||
sqlx::query("DELETE FROM jobs.recoverable_runs WHERE id = $1")
|
||||
.bind(run_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.ok();
|
||||
}
|
||||
|
||||
// ─── Parallel-test serialization ───────────────────────────────────────
|
||||
//
|
||||
// Cargo runs `#[test]` fns in parallel; the three tests in this
|
||||
// module all target `job_name = 'drives_consistency'` in
|
||||
// `jobs.recoverable_runs`. Without serialization,
|
||||
// `second_trigger_is_already_active` seeds a Running row that
|
||||
// makes `detects_used_bytes_drift`'s `open_or_start` short-circuit
|
||||
// with `AlreadyActive` — its handler never dispatches, no
|
||||
// `consistency_finding` fires, and the drift assertion sees empty
|
||||
// events. Holding `TEST_LOCK` for each test's full body prevents
|
||||
// that interleaving; `wipe_our_runs` on entry defends against
|
||||
// stale rows left by a crashed / cancelled prior run.
|
||||
|
||||
static TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
async fn wipe_our_runs(pool: &sqlx::PgPool) {
|
||||
sqlx::query("DELETE FROM jobs.recoverable_runs WHERE job_name = 'drives_consistency'")
|
||||
.execute(pool)
|
||||
.await
|
||||
.ok();
|
||||
}
|
||||
|
||||
// ─── The tests ─────────────────────────────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn drives_consistency_detects_used_bytes_drift() {
|
||||
let _lock = TEST_LOCK.lock().await;
|
||||
let pool = test_pool().await;
|
||||
wipe_our_runs(pool.as_ref()).await;
|
||||
// Cached = 999, actual = 200 → delta = 799 (positive = cached over-reports).
|
||||
let drive_id = seed_drift(pool.as_ref(), 999, 200).await;
|
||||
|
||||
// Run end-to-end through the recoverable engine: PgJobStoreProvider
|
||||
// creates a run row, run_or_resume dispatches DrivesConsistencyCheck,
|
||||
// handler walks the drive, marks Completed. Findings land in
|
||||
// `jobs.run_findings` via `store.record_finding()` — asserted
|
||||
// via `provider.list_findings(run_id, ...)` below (post-Slice 7,
|
||||
// no more tracing-event capture).
|
||||
let provider: Arc<dyn JobStoreProvider> = Arc::new(
|
||||
crate::infrastructure::scheduler::PgJobStoreProvider::new(pool.clone()),
|
||||
);
|
||||
let handler: Arc<dyn RecoverableJobHandler> =
|
||||
Arc::new(DrivesConsistencyCheck::new(pool.clone()));
|
||||
let outcome = crate::infrastructure::scheduler::run_or_resume(
|
||||
handler,
|
||||
provider.clone(),
|
||||
&JobRunArgs::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Framework assertions.
|
||||
assert!(outcome.is_ok(), "run must complete: {outcome:?}");
|
||||
|
||||
// Read-only invariant — drive's used_bytes is UNCHANGED by the check.
|
||||
let post_cached: i64 = sqlx::query("SELECT used_bytes FROM storage.drives WHERE id = $1")
|
||||
.bind(drive_id)
|
||||
.fetch_one(pool.as_ref())
|
||||
.await
|
||||
.expect("drive still exists")
|
||||
.get(0);
|
||||
assert_eq!(post_cached, 999, "drives_consistency must be read-only");
|
||||
|
||||
// Find the run row that was created and verify its state.
|
||||
let latest_run: Option<(Uuid, String, i64)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT id, status, COALESCE((stats->>'scanned_count')::bigint, 0)
|
||||
FROM jobs.recoverable_runs
|
||||
WHERE job_name = 'drives_consistency'
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1
|
||||
"#,
|
||||
)
|
||||
.fetch_optional(pool.as_ref())
|
||||
.await
|
||||
.expect("query recoverable_runs");
|
||||
let (run_id, status, scanned) = latest_run.expect("run row must exist after run_or_resume");
|
||||
assert_eq!(status, "Completed", "run must be Completed");
|
||||
assert!(
|
||||
scanned >= 1,
|
||||
"scanned_count must include at least our drive, got {scanned}"
|
||||
);
|
||||
|
||||
// Drift-detection assertion — the finding for our seeded drive
|
||||
// is now a persisted row. Query via the same interface the
|
||||
// admin endpoint uses so the test also pins the read path.
|
||||
let findings = provider
|
||||
.list_findings(run_id, 500, 0)
|
||||
.await
|
||||
.expect("list_findings");
|
||||
let ours = findings
|
||||
.iter()
|
||||
.find(|f| f.resource_id == Some(drive_id))
|
||||
.unwrap_or_else(|| {
|
||||
panic!("expected a persisted finding for drive {drive_id}, got: {findings:?}")
|
||||
});
|
||||
assert_eq!(ours.kind, "stale_used_bytes", "wrong kind: {ours:?}");
|
||||
assert_eq!(ours.severity, "inconsistent", "wrong severity: {ours:?}");
|
||||
assert_eq!(
|
||||
ours.detail.get("cached").and_then(|v| v.as_i64()),
|
||||
Some(999),
|
||||
"cached mismatch in detail: {ours:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
ours.detail.get("actual").and_then(|v| v.as_i64()),
|
||||
Some(200),
|
||||
"actual mismatch in detail: {ours:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
ours.detail.get("delta").and_then(|v| v.as_i64()),
|
||||
Some(799),
|
||||
"delta mismatch in detail: {ours:?}"
|
||||
);
|
||||
|
||||
// Counter mirror — record_finding also bumps stats.finding_count.
|
||||
let stored_finding_count: i64 = sqlx::query(
|
||||
"SELECT COALESCE((stats->>'finding_count')::bigint, 0) FROM jobs.recoverable_runs WHERE id = $1",
|
||||
)
|
||||
.bind(run_id)
|
||||
.fetch_one(pool.as_ref())
|
||||
.await
|
||||
.expect("query stats.finding_count")
|
||||
.get(0);
|
||||
assert!(
|
||||
stored_finding_count >= 1,
|
||||
"finding_count must be bumped, got {stored_finding_count}"
|
||||
);
|
||||
|
||||
// Cleanup — even on assertion failure the test panics before this,
|
||||
// leaving the test DB slightly dirty. That's fine per session; the
|
||||
// next spawn-db.sh reset clears everything.
|
||||
cleanup_run(pool.as_ref(), run_id).await;
|
||||
cleanup_test_drive(pool.as_ref(), drive_id).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drives_consistency_no_drift_emits_no_finding() {
|
||||
let _lock = TEST_LOCK.lock().await;
|
||||
let pool = test_pool().await;
|
||||
wipe_our_runs(pool.as_ref()).await;
|
||||
// cached == actual → no drift.
|
||||
let drive_id = seed_drift(pool.as_ref(), 500, 500).await;
|
||||
|
||||
let provider: Arc<dyn JobStoreProvider> = Arc::new(
|
||||
crate::infrastructure::scheduler::PgJobStoreProvider::new(pool.clone()),
|
||||
);
|
||||
let handler: Arc<dyn RecoverableJobHandler> =
|
||||
Arc::new(DrivesConsistencyCheck::new(pool.clone()));
|
||||
let outcome = crate::infrastructure::scheduler::run_or_resume(
|
||||
handler,
|
||||
provider.clone(),
|
||||
&JobRunArgs::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(outcome.is_ok());
|
||||
|
||||
// Locate the run row we just wrote.
|
||||
let latest_run: Option<(Uuid,)> = sqlx::query_as(
|
||||
"SELECT id FROM jobs.recoverable_runs WHERE job_name='drives_consistency' ORDER BY started_at DESC LIMIT 1",
|
||||
)
|
||||
.fetch_optional(pool.as_ref())
|
||||
.await
|
||||
.expect("query recoverable_runs");
|
||||
let (run_id,) = latest_run.expect("run row must exist");
|
||||
|
||||
// For THIS drive, no persisted finding. Other drives in the test
|
||||
// DB may still surface findings (unrelated fixture data); we only
|
||||
// assert the invariant scoped to our drive_id.
|
||||
let findings = provider
|
||||
.list_findings(run_id, 500, 0)
|
||||
.await
|
||||
.expect("list_findings");
|
||||
let ours = findings
|
||||
.iter()
|
||||
.filter(|f| f.resource_id == Some(drive_id))
|
||||
.count();
|
||||
assert_eq!(
|
||||
ours, 0,
|
||||
"no drift on this drive, expected 0 findings, got {ours}: {findings:?}"
|
||||
);
|
||||
|
||||
cleanup_run(pool.as_ref(), run_id).await;
|
||||
cleanup_test_drive(pool.as_ref(), drive_id).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drives_consistency_second_trigger_is_already_active() {
|
||||
let _lock = TEST_LOCK.lock().await;
|
||||
let pool = test_pool().await;
|
||||
wipe_our_runs(pool.as_ref()).await;
|
||||
|
||||
// Directly INSERT a Running row for drives_consistency to
|
||||
// simulate an in-flight prior dispatch, then observe that
|
||||
// open_or_start refuses to spawn a parallel run.
|
||||
let seeded_run_id = Uuid::new_v4();
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO jobs.recoverable_runs (id, job_name, status, started_at, last_progress_at)
|
||||
VALUES ($1, 'drives_consistency', 'Running', NOW(), NOW())
|
||||
ON CONFLICT (job_name) WHERE status IN ('Running', 'Paused', 'CancelRequested') DO NOTHING
|
||||
"#,
|
||||
)
|
||||
.bind(seeded_run_id)
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.expect("insert seed Running row");
|
||||
|
||||
let provider: Arc<dyn JobStoreProvider> = Arc::new(
|
||||
crate::infrastructure::scheduler::PgJobStoreProvider::new(pool.clone()),
|
||||
);
|
||||
let opened = provider
|
||||
.open_or_start("drives_consistency")
|
||||
.await
|
||||
.expect("open_or_start");
|
||||
match opened {
|
||||
OpenedRun::AlreadyActive { status, .. } => {
|
||||
assert_eq!(status, RunStatus::Running);
|
||||
}
|
||||
_ => panic!("expected AlreadyActive for a job with a Running row"),
|
||||
}
|
||||
|
||||
// Cleanup.
|
||||
sqlx::query("DELETE FROM jobs.recoverable_runs WHERE job_name = 'drives_consistency'")
|
||||
.execute(pool.as_ref())
|
||||
.await
|
||||
.ok();
|
||||
}
|
||||
}
|
||||
@@ -351,6 +351,29 @@ impl BlobStorageBackend for EncryptedBlobBackend {
|
||||
// Encrypted blobs cannot be served directly from disk
|
||||
None
|
||||
}
|
||||
|
||||
/// Enumeration = plaintext hashes, same as the inner backend.
|
||||
/// Encryption operates on payload bytes, not on the hash key:
|
||||
/// blob objects on the inner backend are stored under the
|
||||
/// PLAINTEXT hash so dedup works. Delegating list to the inner
|
||||
/// backend therefore returns exactly the right identifiers.
|
||||
fn list_blob_hashes(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: usize,
|
||||
) -> Pin<
|
||||
Box<
|
||||
dyn std::future::Future<
|
||||
Output = Result<
|
||||
crate::application::ports::blob_storage_ports::BlobListPage,
|
||||
DomainError,
|
||||
>,
|
||||
> + Send
|
||||
+ '_,
|
||||
>,
|
||||
> {
|
||||
self.inner.list_blob_hashes(cursor, limit)
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect a byte stream into a single `Vec<u8>`.
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
//! Third tenant of Part 2 (recoverable-run engine).
|
||||
//!
|
||||
//! Iterates `storage.files` and reports each row whose parent-folder
|
||||
//! state, blob reference, or denormalised size has drifted from
|
||||
//! what the join with `storage.folders` + `storage.blobs` says is
|
||||
//! true. **Read-only** — the fix path is other jobs (trash cascade
|
||||
//! repair, dedup GC, blob resurrection).
|
||||
//!
|
||||
//! Post-D7 files schema notable columns:
|
||||
//!
|
||||
//! * `folder_id` — nullable; `NULL` = file at drive root. Cascade FK
|
||||
//! to `storage.folders`.
|
||||
//! * `blob_hash` — `NOT NULL`; MUST reference a row in
|
||||
//! `storage.blobs.hash`.
|
||||
//! * `size` — denormalised copy of the blob's byte length; the
|
||||
//! original source of truth is `storage.blobs.size` (upload path
|
||||
//! sets both; a mismatch is drift).
|
||||
//! * NO `path` column and NO `user_id` column (dropped in D7). The
|
||||
//! memory note's "path matches parent chain" check from the
|
||||
//! earlier taxonomy does NOT apply here — files carry no
|
||||
//! materialised path.
|
||||
//!
|
||||
//! ### v1 checks (three per-row branches)
|
||||
//!
|
||||
//! * `parent_folder_trashed` — a live file under a soft-deleted
|
||||
//! parent folder. FK cascade + trash cascade should make this
|
||||
//! impossible; occurrence means the cascade missed the row.
|
||||
//! Files at drive root (`folder_id IS NULL`) are exempt — there is
|
||||
//! no parent to check.
|
||||
//! * `missing_blob` — file's `blob_hash` has no row in
|
||||
//! `storage.blobs`. **Severity `data_loss`**: the file record
|
||||
//! points at bytes the blob table doesn't know about, so any read
|
||||
//! attempt fails. Historically this happens when the dedup GC
|
||||
//! reaped a blob whose ref-count decrement raced with a fresh
|
||||
//! file INSERT — the two-phase mark/sweep is meant to prevent
|
||||
//! this, but the check surfaces regressions immediately.
|
||||
//! * `blob_size_mismatch` — `files.size != blobs.size`. Cheap
|
||||
//! because the same LEFT JOIN already loads `blobs.size`. Would
|
||||
//! indicate the denormalised copy was ever set by a code path that
|
||||
//! didn't read the blob's real length — a bug we want to see fast.
|
||||
//!
|
||||
//! ### Room to grow (same self-join, one more `if`)
|
||||
//!
|
||||
//! * `drive_id_parent_mismatch` — `files.drive_id` differs from
|
||||
//! `parent.drive_id`. The join already loads it; adding this once
|
||||
//! drive-membership rules stabilise post-D7 costs one branch.
|
||||
//! * `mime_type_reconciliation` — compare `files.mime_type` against
|
||||
//! the blob's `content_type`. Requires deciding which is
|
||||
//! authoritative first.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::infrastructure::scheduler::{
|
||||
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome,
|
||||
RunStatus, record_or_log,
|
||||
};
|
||||
|
||||
pub const FILES_CONSISTENCY_JOB_NAME: &str = "files_consistency";
|
||||
|
||||
/// Rows per batch. Files can be very numerous — hundreds of
|
||||
/// thousands on medium installs, millions on large — but the per-row
|
||||
/// work is a couple of comparisons. 500 keeps the cancel-poll cadence
|
||||
/// sub-second while amortising round-trip overhead.
|
||||
const BATCH_SIZE: i64 = 500;
|
||||
|
||||
pub struct FilesConsistencyCheck {
|
||||
pool: Arc<PgPool>,
|
||||
}
|
||||
|
||||
impl FilesConsistencyCheck {
|
||||
pub fn new(pool: Arc<PgPool>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Chainable self-registration — mirrors the other consistency
|
||||
/// tenants. On-demand only (operators trigger from
|
||||
/// `POST /api/admin/jobs/files_consistency/trigger` or via
|
||||
/// `consistency_batch`).
|
||||
pub async fn register_recoverable_job(
|
||||
self: Arc<Self>,
|
||||
registry: &JobRegistry,
|
||||
provider: &Arc<dyn JobStoreProvider>,
|
||||
) -> Arc<Self> {
|
||||
registry
|
||||
.register_recoverable_job(self.clone(), provider.clone(), None)
|
||||
.await;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
struct FileRow {
|
||||
id: Uuid,
|
||||
/// File name (basename). Captured into finding `detail` so
|
||||
/// operators see a human identifier next to the UUID.
|
||||
name: String,
|
||||
folder_id: Option<Uuid>,
|
||||
is_trashed: bool,
|
||||
size: i64,
|
||||
blob_hash: String,
|
||||
/// `None` when `folder_id IS NULL` (file at drive root) — the
|
||||
/// LEFT JOIN yields no parent row.
|
||||
parent_is_trashed: Option<bool>,
|
||||
/// Parent folder's materialised `path` (post-D7 files carry no
|
||||
/// path themselves). `None` for root files.
|
||||
parent_path: Option<String>,
|
||||
/// Legacy whole-file blob row size (pre-CDC). `None` when the
|
||||
/// file was ingested via CDC (`chunk_manifests` path) OR when
|
||||
/// the blob is truly missing — disambiguated by `manifest_size`.
|
||||
blob_size: Option<i64>,
|
||||
/// CDC manifest total size. `Some` when the file was ingested
|
||||
/// via FastCDC (its bytes live as chunks referenced by
|
||||
/// `storage.chunk_manifests.chunk_hashes`, not as one
|
||||
/// `storage.blobs` row). `None` when there is no manifest for
|
||||
/// this hash.
|
||||
manifest_size: Option<i64>,
|
||||
/// Total chunks the manifest claims. `None` when the file is
|
||||
/// pre-CDC (whole-file blob path) or has no manifest.
|
||||
manifest_chunk_count: Option<i32>,
|
||||
/// Count of chunks referenced by the manifest that have NO
|
||||
/// matching row in `storage.blobs`. `None` when there's no
|
||||
/// manifest to check. `Some(n)` with `n > 0` means the manifest
|
||||
/// points at reaped chunks — a real data-loss condition, more
|
||||
/// precise than plain `missing_blob` (which only fires when the
|
||||
/// whole-file registry entry is absent). This is a DB-registry
|
||||
/// check; physical backend-existence checks belong in the
|
||||
/// future `storage_consistency` tenant.
|
||||
chunks_missing: Option<i64>,
|
||||
}
|
||||
|
||||
/// Build the file's display path from its folder's `path` and its
|
||||
/// own `name`. Root files just show the name. Trashed folder paths
|
||||
/// still work (ltree keeps them intact under `is_trashed`).
|
||||
fn display_path(folder_path: Option<&str>, name: &str) -> String {
|
||||
match folder_path {
|
||||
Some(p) if !p.is_empty() => format!("{p}/{name}"),
|
||||
_ => name.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RecoverableJobHandler for FilesConsistencyCheck {
|
||||
fn name(&self) -> &str {
|
||||
FILES_CONSISTENCY_JOB_NAME
|
||||
}
|
||||
|
||||
/// Definitive count — one row per file. This is the largest table
|
||||
/// of the trio (millions on big installs); COUNT(*) is still an
|
||||
/// index-only scan but can take ~seconds. The tradeoff is worth
|
||||
/// it — an operator staring at a running files_consistency scan
|
||||
/// wants a bar, and a seconds-scale one-off at run start is
|
||||
/// invisible compared to the multi-minute scan that follows.
|
||||
async fn count_total(&self) -> Option<u64> {
|
||||
let row: Result<(i64,), sqlx::Error> = sqlx::query_as("SELECT COUNT(*) FROM storage.files")
|
||||
.fetch_one(self.pool.as_ref())
|
||||
.await;
|
||||
match row {
|
||||
Ok((n,)) => Some(n.max(0) as u64),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "files_consistency.count_total_failed",
|
||||
error = %e,
|
||||
"count_total failed — run will not surface a progress bar"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_resumable(
|
||||
&self,
|
||||
store: &dyn JobStore,
|
||||
_args: &JobRunArgs,
|
||||
resume_cursor: Option<Vec<u8>>,
|
||||
) -> RunOutcome {
|
||||
// Cursor: 16 raw UUID bytes, empty/absent = start from
|
||||
// beginning. Same convention as the other UUID-cursor
|
||||
// tenants so the resume path in `PgJobStoreProvider` is
|
||||
// uniform.
|
||||
let mut cursor: Option<Uuid> = match resume_cursor {
|
||||
None => None,
|
||||
Some(bytes) if bytes.is_empty() => None,
|
||||
Some(bytes) if bytes.len() == 16 => {
|
||||
let mut arr = [0u8; 16];
|
||||
arr.copy_from_slice(&bytes);
|
||||
Some(Uuid::from_bytes(arr))
|
||||
}
|
||||
Some(bytes) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("invalid cursor: expected 16 bytes, got {}", bytes.len()),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let mut finding_count = 0u64;
|
||||
|
||||
loop {
|
||||
// Cancel poll BETWEEN batches — the cooperative cancel
|
||||
// contract (see `RecoverableJobHandler` trait doc).
|
||||
match store.status().await {
|
||||
Ok(RunStatus::CancelRequested) => {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "files_consistency.cancelled",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"files_consistency cancelled cooperatively, pausing"
|
||||
);
|
||||
return RunOutcome::Paused {
|
||||
cursor: cursor.map(|u| u.as_bytes().to_vec()).unwrap_or_default(),
|
||||
};
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("status poll: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// One query, two LEFT JOINs: (parent folder) + (blob
|
||||
// row). Left-joining the blob is what lets us detect
|
||||
// `missing_blob` — a matched row has `blob.size`
|
||||
// populated; a miss surfaces as NULL.
|
||||
// Three LEFT JOINs — the blob-existence check has to
|
||||
// handle BOTH storage paths OxiCloud uses:
|
||||
//
|
||||
// * `storage.chunk_manifests` (CDC / FastCDC) — the
|
||||
// dominant path for anything ingested after Apr 2026.
|
||||
// Whole-file hash lives here; actual bytes are chunks
|
||||
// referenced by `chunk_hashes[]`.
|
||||
// * `storage.blobs` (legacy pre-CDC whole-file blob) —
|
||||
// still supported via the read path's fallback for
|
||||
// pre-CDC uploads.
|
||||
//
|
||||
// A file is "missing_blob" ONLY when NEITHER row exists.
|
||||
// Deep chunk validation (every chunk in `chunk_hashes[]`
|
||||
// present in `storage.blobs`) is out of scope here — it
|
||||
// belongs in the future `storage_consistency` tenant that
|
||||
// walks the backend against the blob registry.
|
||||
// Correlated subquery `chunks_missing` runs per-row over
|
||||
// the manifest's chunk_hashes array. `hash` is indexed
|
||||
// (PRIMARY KEY on storage.blobs), so each `NOT EXISTS`
|
||||
// probe is O(log n). NULL (not zero) when the file is
|
||||
// pre-CDC or has no manifest — the LEFT JOIN result on
|
||||
// `m` is NULL and `unnest(NULL::text[])` yields zero rows.
|
||||
let rows: Vec<FileRow> = match sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
f.id AS id,
|
||||
f.name AS name,
|
||||
f.folder_id AS folder_id,
|
||||
f.is_trashed AS is_trashed,
|
||||
f.size AS size,
|
||||
f.blob_hash AS blob_hash,
|
||||
parent.is_trashed AS parent_is_trashed,
|
||||
parent.path AS parent_path,
|
||||
b.size AS blob_size,
|
||||
m.total_size AS manifest_size,
|
||||
m.chunk_count AS manifest_chunk_count,
|
||||
CASE WHEN m.chunk_hashes IS NULL THEN NULL
|
||||
ELSE (
|
||||
SELECT COUNT(*)::bigint
|
||||
FROM unnest(m.chunk_hashes) AS ch(hash)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM storage.blobs bb
|
||||
WHERE bb.hash = ch.hash
|
||||
)
|
||||
)
|
||||
END AS chunks_missing
|
||||
FROM storage.files f
|
||||
LEFT JOIN storage.folders parent ON parent.id = f.folder_id
|
||||
LEFT JOIN storage.blobs b ON b.hash = f.blob_hash
|
||||
LEFT JOIN storage.chunk_manifests m ON m.file_hash = f.blob_hash
|
||||
WHERE ($1::uuid IS NULL OR f.id > $1)
|
||||
-- Grace: skip files < 1h old. Delta-upload inserts
|
||||
-- chunks with ref_count=0 BEFORE the commit that
|
||||
-- inserts the file row + manifest, so the normal
|
||||
-- path is race-free — but replace/overwrite flows
|
||||
-- have narrow windows where a mid-transaction scan
|
||||
-- could see `missing_blob` or `chunk_missing`.
|
||||
-- Same grace shape as `blobs_consistency`.
|
||||
AND f.created_at < NOW() - INTERVAL '1 hour'
|
||||
ORDER BY f.id
|
||||
LIMIT $2
|
||||
"#,
|
||||
)
|
||||
.bind(cursor)
|
||||
.bind(BATCH_SIZE)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("batch fetch: {e}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if rows.is_empty() {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "files_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"files_consistency completed with {} finding(s)",
|
||||
finding_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
|
||||
for row in &rows {
|
||||
// Human-readable path captured once per row and folded
|
||||
// into every finding on this row. `name` is the raw
|
||||
// basename (useful even when the parent is orphaned
|
||||
// and `parent_path` is None).
|
||||
let path = display_path(row.parent_path.as_deref(), &row.name);
|
||||
|
||||
// (1) parent_folder_trashed: live file under a
|
||||
// soft-deleted folder. Root files (`folder_id IS
|
||||
// NULL`) are exempt — `parent_is_trashed` is None
|
||||
// there.
|
||||
if !row.is_trashed && row.parent_is_trashed == Some(true) {
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
FILES_CONSISTENCY_JOB_NAME,
|
||||
"parent_folder_trashed",
|
||||
"inconsistent",
|
||||
Some(row.id),
|
||||
serde_json::json!({
|
||||
"name": row.name,
|
||||
"path": path,
|
||||
"folder_id": row.folder_id,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Content-bearing size for this file, in priority
|
||||
// order: CDC manifest (dominant path — every file
|
||||
// uploaded after Apr 2026), then legacy pre-CDC
|
||||
// whole-file blob. `None` = no registry entry on
|
||||
// either path → real `missing_blob`.
|
||||
let content_size = row.manifest_size.or(row.blob_size);
|
||||
|
||||
// (2) missing_blob: NEITHER the CDC manifest nor the
|
||||
// legacy blob row exists for this hash. Real data-loss
|
||||
// indicator — the read path checks manifest first and
|
||||
// falls back to blob; if both are missing, reading
|
||||
// the file will fail. NOT a false positive for CDC
|
||||
// files, because the manifest check catches them.
|
||||
if content_size.is_none() {
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
FILES_CONSISTENCY_JOB_NAME,
|
||||
"missing_blob",
|
||||
"data_loss",
|
||||
Some(row.id),
|
||||
serde_json::json!({
|
||||
"name": row.name,
|
||||
"path": path,
|
||||
"blob_hash": row.blob_hash,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
// No point checking size when neither registry
|
||||
// entry exists — skip (3) for this row.
|
||||
continue;
|
||||
}
|
||||
|
||||
// (2b) chunk_missing: the file's CDC manifest exists
|
||||
// and points at N chunks, but K of them have no row
|
||||
// in `storage.blobs`. Real data-loss condition — the
|
||||
// read path will fail reassembly when it tries to
|
||||
// fetch a reaped chunk. Typically caused by a dedup
|
||||
// GC race (chunk reaped while a manifest still held
|
||||
// a reference) or partial pg_dump/restore that
|
||||
// dropped `storage.blobs` rows.
|
||||
if let Some(missing) = row.chunks_missing
|
||||
&& missing > 0
|
||||
{
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
FILES_CONSISTENCY_JOB_NAME,
|
||||
"chunk_missing",
|
||||
"data_loss",
|
||||
Some(row.id),
|
||||
serde_json::json!({
|
||||
"name": row.name,
|
||||
"path": path,
|
||||
"blob_hash": row.blob_hash,
|
||||
"chunks_missing": missing,
|
||||
"chunks_total": row.manifest_chunk_count,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
// Deliberately DON'T `continue` — a
|
||||
// chunk_missing finding does not preclude a
|
||||
// size mismatch, and the two are independent
|
||||
// signals worth surfacing separately.
|
||||
}
|
||||
|
||||
// (3) blob_size_mismatch: denormalised size drifted
|
||||
// from the content-registry's authoritative size.
|
||||
// Prefers manifest.total_size when present (post-CDC
|
||||
// ingest path); falls back to blob.size (legacy).
|
||||
if let Some(bs) = content_size
|
||||
&& bs != row.size
|
||||
{
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
FILES_CONSISTENCY_JOB_NAME,
|
||||
"blob_size_mismatch",
|
||||
"inconsistent",
|
||||
Some(row.id),
|
||||
serde_json::json!({
|
||||
"name": row.name,
|
||||
"path": path,
|
||||
"blob_hash": row.blob_hash,
|
||||
"stored": row.size,
|
||||
"actual": bs,
|
||||
"delta": row.size - bs,
|
||||
"source": if row.manifest_size.is_some() { "manifest" } else { "blob" },
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// (4) legacy_uncdc_file: informational — this file
|
||||
// is served via the pre-CDC whole-file blob fallback,
|
||||
// not the modern chunk-manifest path. Not broken;
|
||||
// just misses out on sub-file dedup benefits + never
|
||||
// shares chunks with newer uploads. Severity
|
||||
// `anomaly` (surprising state, no known impact) so
|
||||
// the UI renders it as a notice, not a warning.
|
||||
// Fires when the blob registry has a whole-file row
|
||||
// for this hash but there is no manifest. Recovery
|
||||
// path: `ReingestLegacy` (deferred, see
|
||||
// `docs/plan/recovery.md`).
|
||||
if row.blob_size.is_some() && row.manifest_size.is_none() {
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
FILES_CONSISTENCY_JOB_NAME,
|
||||
"legacy_uncdc_file",
|
||||
"anomaly",
|
||||
Some(row.id),
|
||||
serde_json::json!({
|
||||
"name": row.name,
|
||||
"path": path,
|
||||
"blob_hash": row.blob_hash,
|
||||
"size": row.size,
|
||||
"note": "pre-CDC whole-file blob; re-ingest for sub-file dedup",
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
// Advance cursor + checkpoint. `batch_len` feeds
|
||||
// `stats.scanned_count`.
|
||||
let last_id = rows.last().map(|r| r.id).expect("non-empty rows");
|
||||
cursor = Some(last_id);
|
||||
let batch_len = rows.len() as u64;
|
||||
if let Err(e) = store
|
||||
.checkpoint(last_id.as_bytes().to_vec(), batch_len)
|
||||
.await
|
||||
{
|
||||
return RunOutcome::Failed {
|
||||
message: format!("checkpoint: {e}"),
|
||||
};
|
||||
}
|
||||
|
||||
if (rows.len() as i64) < BATCH_SIZE {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "files_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"files_consistency completed with {} finding(s)",
|
||||
finding_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
//! Second tenant of Part 2 (recoverable-run engine).
|
||||
//!
|
||||
//! Iterates `storage.folders` and reports each row whose maintained-
|
||||
//! by-trigger materialised state has drifted from what walking its
|
||||
//! `parent_id` chain would produce. **Read-only** — reports drift as
|
||||
//! findings but does NOT rewrite anything; a curative sweep (or a
|
||||
//! targeted repair endpoint) is a separate concern.
|
||||
//!
|
||||
//! Why this matters: the ltree cascade trigger `trg_folders_cascade_path`
|
||||
//! is what keeps `path` and `lpath` in sync with `parent_id`. Any
|
||||
//! bulk write path that bypasses the trigger (raw COPY, migration
|
||||
//! backfill with FOR EACH STATEMENT triggers disabled, hand-rolled
|
||||
//! UPDATE with `SET LOCAL session_replication_role = 'replica'`) can
|
||||
//! leave the materialised columns wrong. Silent divergence breaks
|
||||
//! every `WHERE lpath <@ ancestor` query — subtree list, breadcrumb
|
||||
//! walk, recursive copy/move/delete. Detecting the drift is what
|
||||
//! surfaces the underlying misuse.
|
||||
//!
|
||||
//! ### v1 checks (three per-row branches)
|
||||
//!
|
||||
//! * `parent_trashed_mismatch` — a non-trashed folder whose parent
|
||||
//! IS trashed. FK enforcement + trash cascade should make this
|
||||
//! impossible; when it does happen the cascade missed a row.
|
||||
//! * `path_mismatch` — materialised `folders.path` differs from
|
||||
//! the parent-chain reconstruction.
|
||||
//! * `lpath_mismatch` — materialised `folders.lpath` differs from
|
||||
//! the parent-chain reconstruction.
|
||||
//!
|
||||
//! Reconstruction convention (mirrors `storage.compute_folder_path`
|
||||
//! from `20260307000000_initial_schema.sql`):
|
||||
//!
|
||||
//! ```text
|
||||
//! my_label = replace(id::text, '-', '_')
|
||||
//! root: path = name lpath = my_label
|
||||
//! non-root: path = parent.path || '/' || name
|
||||
//! lpath = parent.lpath || my_label
|
||||
//! ```
|
||||
//!
|
||||
//! Findings are LOGGED to `target: "oxicloud::consistency"` for now,
|
||||
//! same as `drives_consistency`. Persistence to `jobs.run_findings`
|
||||
//! lands with the findings-table migration (deferred); at that point
|
||||
//! this handler swaps its `tracing::warn!` calls for
|
||||
//! `store.record_finding(...)` — nothing else changes.
|
||||
//!
|
||||
//! ### Room to grow (already-cheap branches deferred)
|
||||
//!
|
||||
//! * `drive_id_parent_mismatch` — parent + child in different drives.
|
||||
//! The self-join already loads `parent.drive_id`; one more per-row
|
||||
//! `if` when the drive-membership rules stabilise post-D7.
|
||||
//! * `orphan_root` — a non-trashed `parent_id IS NULL` folder no
|
||||
//! `drives.root_folder_id` points at. The `check_no_orphan_root_folder`
|
||||
//! trigger from `20260803000000_*` blocks new orphans; a check here
|
||||
//! would surface pre-trigger legacy rows.
|
||||
//! * Name-vs-path terminal drift (`path` ending in the folder's `name`).
|
||||
//! Redundant with `path_mismatch` unless we ever start storing them
|
||||
//! independently.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::infrastructure::scheduler::{
|
||||
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome,
|
||||
RunStatus, record_or_log,
|
||||
};
|
||||
|
||||
pub const FOLDERS_CONSISTENCY_JOB_NAME: &str = "folders_consistency";
|
||||
|
||||
/// Rows per batch. Folders can be numerous (millions on big
|
||||
/// installs), each row is light. 500 keeps the cancel-poll cadence
|
||||
/// sub-second on a warm cache while amortising round-trip overhead.
|
||||
const BATCH_SIZE: i64 = 500;
|
||||
|
||||
pub struct FoldersConsistencyCheck {
|
||||
pool: Arc<PgPool>,
|
||||
}
|
||||
|
||||
impl FoldersConsistencyCheck {
|
||||
pub fn new(pool: Arc<PgPool>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Chainable self-registration — mirrors `DrivesConsistencyCheck`.
|
||||
/// On-demand only (no periodic tick); operators fire it from
|
||||
/// `POST /api/admin/jobs/folders_consistency/trigger`.
|
||||
pub async fn register_recoverable_job(
|
||||
self: Arc<Self>,
|
||||
registry: &JobRegistry,
|
||||
provider: &Arc<dyn JobStoreProvider>,
|
||||
) -> Arc<Self> {
|
||||
registry
|
||||
.register_recoverable_job(self.clone(), provider.clone(), None)
|
||||
.await;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
struct FolderRow {
|
||||
id: Uuid,
|
||||
/// Folder basename — surfaced in finding `detail` so operators
|
||||
/// see a human identifier next to the UUID.
|
||||
name: String,
|
||||
parent_id: Option<Uuid>,
|
||||
is_trashed: bool,
|
||||
path: String,
|
||||
lpath_text: String,
|
||||
parent_is_trashed: Option<bool>,
|
||||
parent_path: Option<String>,
|
||||
parent_lpath_text: Option<String>,
|
||||
expected_path: String,
|
||||
expected_lpath_text: String,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RecoverableJobHandler for FoldersConsistencyCheck {
|
||||
fn name(&self) -> &str {
|
||||
FOLDERS_CONSISTENCY_JOB_NAME
|
||||
}
|
||||
|
||||
/// Definitive count — one row per folder. Larger table than drives
|
||||
/// but the COUNT(*) is still index-only on PG. On multi-million-row
|
||||
/// deployments this is ~100ms at run start; acceptable given the
|
||||
/// progress bar is only rendered when the operator is watching.
|
||||
async fn count_total(&self) -> Option<u64> {
|
||||
let row: Result<(i64,), sqlx::Error> =
|
||||
sqlx::query_as("SELECT COUNT(*) FROM storage.folders")
|
||||
.fetch_one(self.pool.as_ref())
|
||||
.await;
|
||||
match row {
|
||||
Ok((n,)) => Some(n.max(0) as u64),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "folders_consistency.count_total_failed",
|
||||
error = %e,
|
||||
"count_total failed — run will not surface a progress bar"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_resumable(
|
||||
&self,
|
||||
store: &dyn JobStore,
|
||||
_args: &JobRunArgs,
|
||||
resume_cursor: Option<Vec<u8>>,
|
||||
) -> RunOutcome {
|
||||
// Cursor: 16 raw UUID bytes, empty/absent = start from beginning.
|
||||
// Same convention as `drives_consistency` so the resume path
|
||||
// in `PgJobStoreProvider` treats every UUID-cursor tenant the
|
||||
// same way.
|
||||
let mut cursor: Option<Uuid> = match resume_cursor {
|
||||
None => None,
|
||||
Some(bytes) if bytes.is_empty() => None,
|
||||
Some(bytes) if bytes.len() == 16 => {
|
||||
let mut arr = [0u8; 16];
|
||||
arr.copy_from_slice(&bytes);
|
||||
Some(Uuid::from_bytes(arr))
|
||||
}
|
||||
Some(bytes) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("invalid cursor: expected 16 bytes, got {}", bytes.len()),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let mut finding_count = 0u64;
|
||||
|
||||
loop {
|
||||
// Cancel poll BETWEEN batches — the cooperative cancel
|
||||
// contract (see `RecoverableJobHandler` trait doc).
|
||||
match store.status().await {
|
||||
Ok(RunStatus::CancelRequested) => {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "folders_consistency.cancelled",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"folders_consistency cancelled cooperatively, pausing"
|
||||
);
|
||||
return RunOutcome::Paused {
|
||||
cursor: cursor.map(|u| u.as_bytes().to_vec()).unwrap_or_default(),
|
||||
};
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("status poll: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch a batch of folders + reconstruct expected
|
||||
// path/lpath in SQL via LEFT JOIN on parent. The
|
||||
// reconstruction formula mirrors `compute_folder_path`
|
||||
// 1:1; if that trigger's convention ever changes both
|
||||
// must move together.
|
||||
let rows: Vec<FolderRow> = match sqlx::query_as(
|
||||
r#"
|
||||
SELECT
|
||||
f.id AS id,
|
||||
f.name AS name,
|
||||
f.parent_id AS parent_id,
|
||||
f.is_trashed AS is_trashed,
|
||||
f.path AS path,
|
||||
f.lpath::text AS lpath_text,
|
||||
parent.is_trashed AS parent_is_trashed,
|
||||
parent.path AS parent_path,
|
||||
parent.lpath::text AS parent_lpath_text,
|
||||
CASE
|
||||
WHEN f.parent_id IS NULL THEN f.name
|
||||
ELSE parent.path || '/' || f.name
|
||||
END AS expected_path,
|
||||
CASE
|
||||
WHEN f.parent_id IS NULL THEN replace(f.id::text, '-', '_')
|
||||
ELSE parent.lpath::text || '.' || replace(f.id::text, '-', '_')
|
||||
END AS expected_lpath_text
|
||||
FROM storage.folders f
|
||||
LEFT JOIN storage.folders parent ON parent.id = f.parent_id
|
||||
WHERE ($1::uuid IS NULL OR f.id > $1)
|
||||
-- Grace: skip folders < 1h old. The
|
||||
-- `trg_folders_cascade_path` trigger runs on the
|
||||
-- writer's transaction, so a folder created RIGHT
|
||||
-- NOW could momentarily show a `path_mismatch`
|
||||
-- window before the cascade lands. Same grace
|
||||
-- shape as `blobs_consistency`.
|
||||
AND f.created_at < NOW() - INTERVAL '1 hour'
|
||||
ORDER BY f.id
|
||||
LIMIT $2
|
||||
"#,
|
||||
)
|
||||
.bind(cursor)
|
||||
.bind(BATCH_SIZE)
|
||||
.fetch_all(self.pool.as_ref())
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("batch fetch: {e}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if rows.is_empty() {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "folders_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"folders_consistency completed with {} finding(s)",
|
||||
finding_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
|
||||
// Per-row branches. Add new ones here — same pattern as
|
||||
// `drives_consistency`. Emit at most one finding per
|
||||
// (row, kind); multiple different kinds for the same row
|
||||
// are fine and independent.
|
||||
for row in &rows {
|
||||
// (1) parent_trashed_mismatch: a live folder under a
|
||||
// soft-deleted parent. Cascade missed.
|
||||
if !row.is_trashed && row.parent_is_trashed == Some(true) {
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
FOLDERS_CONSISTENCY_JOB_NAME,
|
||||
"parent_trashed_mismatch",
|
||||
"inconsistent",
|
||||
Some(row.id),
|
||||
serde_json::json!({
|
||||
"name": row.name,
|
||||
"path": row.path,
|
||||
"parent_id": row.parent_id,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// (2) path_mismatch: materialised path drifted from
|
||||
// the parent-chain reconstruction.
|
||||
if row.path != row.expected_path {
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
FOLDERS_CONSISTENCY_JOB_NAME,
|
||||
"path_mismatch",
|
||||
"inconsistent",
|
||||
Some(row.id),
|
||||
serde_json::json!({
|
||||
"name": row.name,
|
||||
"stored": row.path,
|
||||
"expected": row.expected_path,
|
||||
"parent_path": row.parent_path,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// (3) lpath_mismatch: materialised lpath drifted from
|
||||
// the parent-chain reconstruction. Independent of (2)
|
||||
// — either can be wrong without the other, and both
|
||||
// silently break different query shapes.
|
||||
if row.lpath_text != row.expected_lpath_text {
|
||||
finding_count += 1;
|
||||
record_or_log(
|
||||
store,
|
||||
FOLDERS_CONSISTENCY_JOB_NAME,
|
||||
"lpath_mismatch",
|
||||
"inconsistent",
|
||||
Some(row.id),
|
||||
serde_json::json!({
|
||||
"name": row.name,
|
||||
"path": row.path,
|
||||
"stored": row.lpath_text,
|
||||
"expected": row.expected_lpath_text,
|
||||
"parent_lpath": row.parent_lpath_text,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
// Advance cursor + checkpoint. Batch length is what we
|
||||
// report to `stats.scanned_count`; findings are separate
|
||||
// (they arrive via the tracing subscriber / eventually
|
||||
// `run_findings`).
|
||||
let last_id = rows.last().map(|r| r.id).expect("non-empty rows");
|
||||
cursor = Some(last_id);
|
||||
let batch_len = rows.len() as u64;
|
||||
if let Err(e) = store
|
||||
.checkpoint(last_id.as_bytes().to_vec(), batch_len)
|
||||
.await
|
||||
{
|
||||
return RunOutcome::Failed {
|
||||
message: format!("checkpoint: {e}"),
|
||||
};
|
||||
}
|
||||
|
||||
// Short batch = drained the folders table.
|
||||
if (rows.len() as i64) < BATCH_SIZE {
|
||||
tracing::info!(
|
||||
target: "oxicloud::consistency",
|
||||
event = "folders_consistency.completed",
|
||||
run_id = %store.run_id(),
|
||||
finding_count = finding_count,
|
||||
"folders_consistency completed with {} finding(s)",
|
||||
finding_count
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ use tracing::{error, info};
|
||||
|
||||
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
||||
use crate::common::errors::DomainError;
|
||||
use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRunArgs};
|
||||
use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs};
|
||||
use crate::infrastructure::services::pg_acl_engine::PgAclEngine;
|
||||
use async_trait::async_trait;
|
||||
|
||||
@@ -57,12 +57,22 @@ impl GrantCleanupService {
|
||||
self.grace_days
|
||||
}
|
||||
|
||||
/// Cadence exposed as `Duration` so DI passes a sanitised value
|
||||
/// (post-`.max(1)`) to `JobRegistry::register`.
|
||||
/// Cadence exposed as `Duration`. Internal helper used by
|
||||
/// [`Self::register`]; kept `pub` for tests.
|
||||
pub fn interval(&self) -> Duration {
|
||||
Duration::from_secs(self.interval_hours * 3600)
|
||||
}
|
||||
|
||||
/// Register self with the periodic-job scheduler and return the
|
||||
/// same `Arc<Self>` for DI-style chaining. Scheduled tenant with
|
||||
/// interval = `self.interval()`, no timeout. See
|
||||
/// `docs/plan/job-registry.md` Part 1.
|
||||
pub async fn register(self: Arc<Self>, registry: &JobRegistry) -> Arc<Self> {
|
||||
let interval = self.interval();
|
||||
registry.register(self.clone(), Some(interval), None).await;
|
||||
self
|
||||
}
|
||||
|
||||
/// Run one purge pass.
|
||||
///
|
||||
/// `grace_override`:
|
||||
|
||||
@@ -10,6 +10,7 @@ use tokio::io::{AsyncSeekExt, AsyncWriteExt};
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use bytes::Bytes;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use crate::application::ports::blob_storage_ports::{
|
||||
BlobStorageBackend, BlobStream, StorageHealthStatus,
|
||||
@@ -607,6 +608,165 @@ impl BlobStorageBackend for LocalBlobBackend {
|
||||
fn read_prefetch(&self) -> usize {
|
||||
self.read_prefetch
|
||||
}
|
||||
|
||||
/// Enumerate `.blob` files under `.blobs/<xx>/`. Cursor format:
|
||||
///
|
||||
/// * `None` — start from the first shard (`00`) at file offset 0
|
||||
/// * `Some("<shard>/<hash>")` — resume: skip shards `< shard`
|
||||
/// entirely, and within `shard` skip files whose hash `≤ hash`.
|
||||
///
|
||||
/// Ordering: shards ascending (00–ff), files within a shard
|
||||
/// ascending by hash. Stable across calls given the sorting.
|
||||
///
|
||||
/// Filter: basename must be exactly 64 hex chars + `.blob`. This
|
||||
/// excludes `.tmp` staging files, `.orig`/`.lost`/`.corrupt`
|
||||
/// sidecars from manual admin work, and any other non-canonical
|
||||
/// artefacts. Backend consistency scans the DB-registered
|
||||
/// content-addressable set only.
|
||||
fn list_blob_hashes(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: usize,
|
||||
) -> Pin<
|
||||
Box<
|
||||
dyn std::future::Future<
|
||||
Output = Result<
|
||||
crate::application::ports::blob_storage_ports::BlobListPage,
|
||||
DomainError,
|
||||
>,
|
||||
> + Send
|
||||
+ '_,
|
||||
>,
|
||||
> {
|
||||
use crate::application::ports::blob_storage_ports::{
|
||||
BackendBlobEntry, BackendUnknownEntry, BlobListPage,
|
||||
};
|
||||
|
||||
let blob_root = self.blob_root.clone();
|
||||
Box::pin(async move {
|
||||
let (start_shard, start_after_hash): (String, Option<String>) = match cursor {
|
||||
None => (String::from("00"), None),
|
||||
Some(c) => match c.split_once('/') {
|
||||
Some((sh, h)) => (sh.to_string(), Some(h.to_string())),
|
||||
None => (c, None),
|
||||
},
|
||||
};
|
||||
|
||||
let mut blobs: Vec<BackendBlobEntry> = Vec::with_capacity(limit);
|
||||
let mut unknowns: Vec<BackendUnknownEntry> = Vec::new();
|
||||
let mut next_cursor: Option<String> = None;
|
||||
|
||||
for prefix in &HEX_PREFIXES {
|
||||
let prefix = *prefix;
|
||||
if prefix < start_shard.as_str() {
|
||||
continue;
|
||||
}
|
||||
let shard_dir = blob_root.join(prefix);
|
||||
let mut entries = match fs::read_dir(&shard_dir).await {
|
||||
Ok(e) => e,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(e) => {
|
||||
return Err(DomainError::new(
|
||||
ErrorKind::InternalError,
|
||||
"Blob",
|
||||
format!("read shard {prefix}: {e}"),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Collect canonical blobs + unknowns for this shard.
|
||||
// The distinction is filename shape: `<64-hex>.blob`
|
||||
// → canonical blob; anything else → unknown sidecar.
|
||||
// Unknowns are captured with their full basename so
|
||||
// the tenant can surface them to operators as
|
||||
// informational notices (severity `anomaly`).
|
||||
let mut shard_blobs: Vec<(String, Option<DateTime<Utc>>)> = Vec::new();
|
||||
let mut shard_unknowns: Vec<(String, Option<DateTime<Utc>>)> = Vec::new();
|
||||
while let Some(dirent) = entries.next_entry().await.map_err(|e| {
|
||||
DomainError::new(
|
||||
ErrorKind::InternalError,
|
||||
"Blob",
|
||||
format!("read shard {prefix} entry: {e}"),
|
||||
)
|
||||
})? {
|
||||
let name = dirent.file_name();
|
||||
let name_str = match name.to_str() {
|
||||
Some(s) => s,
|
||||
None => continue, // non-UTF8 filename — skip entirely
|
||||
};
|
||||
// Skip directories — the shard dir itself
|
||||
// shouldn't contain any, but defensively.
|
||||
if dirent
|
||||
.file_type()
|
||||
.await
|
||||
.map(|t| t.is_dir())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mtime = dirent
|
||||
.metadata()
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|m| m.modified().ok())
|
||||
.map(DateTime::<Utc>::from);
|
||||
|
||||
// Canonical shape check: `<64-hex>.blob`.
|
||||
let canonical = name_str
|
||||
.strip_suffix(".blob")
|
||||
.filter(|stem| {
|
||||
stem.len() == 64 && stem.chars().all(|c| c.is_ascii_hexdigit())
|
||||
})
|
||||
.map(|s| s.to_string());
|
||||
|
||||
match canonical {
|
||||
Some(hash) => shard_blobs.push((hash, mtime)),
|
||||
None => shard_unknowns.push((name_str.to_string(), mtime)),
|
||||
}
|
||||
}
|
||||
shard_blobs.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
// Unknowns don't need cursor-precise ordering — they
|
||||
// ride alongside the blobs batch. Sort just for
|
||||
// stable operator-facing output.
|
||||
shard_unknowns.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
for (name, mtime) in shard_unknowns {
|
||||
unknowns.push(BackendUnknownEntry {
|
||||
path: format!("{prefix}/{name}"),
|
||||
mtime,
|
||||
});
|
||||
}
|
||||
|
||||
for (hash, mtime) in shard_blobs {
|
||||
if prefix == start_shard.as_str()
|
||||
&& let Some(ref after) = start_after_hash
|
||||
&& hash.as_str() <= after.as_str()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if blobs.len() >= limit {
|
||||
next_cursor = Some(format!(
|
||||
"{}/{}",
|
||||
prefix,
|
||||
blobs.last().map(|e| e.hash.as_str()).unwrap_or("")
|
||||
));
|
||||
return Ok(BlobListPage {
|
||||
blobs,
|
||||
unknowns,
|
||||
next_cursor,
|
||||
});
|
||||
}
|
||||
blobs.push(BackendBlobEntry { hash, mtime });
|
||||
}
|
||||
}
|
||||
|
||||
Ok(BlobListPage {
|
||||
blobs,
|
||||
unknowns,
|
||||
next_cursor,
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -229,4 +229,45 @@ impl BlobStorageBackend for MigrationBlobBackend {
|
||||
.local_blob_path(hash)
|
||||
.or_else(|| self.source.local_blob_path(hash))
|
||||
}
|
||||
|
||||
/// Enumeration during migration is intentionally REFUSED. Both
|
||||
/// source and target legitimately hold bytes concurrently
|
||||
/// mid-migration: a blob copied to target but not yet deleted
|
||||
/// from source would be reported "twice"; a blob in-flight from
|
||||
/// source to target could be flagged as orphan on whichever
|
||||
/// side the consistency scan doesn't walk. There's no single
|
||||
/// authoritative "what's on the backend" answer while a
|
||||
/// migration is running.
|
||||
///
|
||||
/// Operators wanting to run `backend_consistency` during a
|
||||
/// migration should either wait for the migration to complete
|
||||
/// (target becomes authoritative) or cancel it. The
|
||||
/// `operation_not_supported` error is surfaced by the tenant as
|
||||
/// a single run-level `backend_unenumerable` finding — no
|
||||
/// per-blob probes attempted.
|
||||
fn list_blob_hashes(
|
||||
&self,
|
||||
_cursor: Option<String>,
|
||||
_limit: usize,
|
||||
) -> Pin<
|
||||
Box<
|
||||
dyn std::future::Future<
|
||||
Output = Result<
|
||||
crate::application::ports::blob_storage_ports::BlobListPage,
|
||||
DomainError,
|
||||
>,
|
||||
> + Send
|
||||
+ '_,
|
||||
>,
|
||||
> {
|
||||
Box::pin(async {
|
||||
Err(DomainError::operation_not_supported(
|
||||
"list_blob_hashes",
|
||||
"backend_consistency cannot enumerate while a storage \
|
||||
migration is in progress — source and target hold bytes \
|
||||
concurrently; wait for migration completion or cancel it \
|
||||
before running the scan",
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
pub mod audio_metadata_service;
|
||||
pub mod azure_blob_backend;
|
||||
pub mod backend_consistency_service;
|
||||
pub mod blobs_consistency_service;
|
||||
pub mod cached_blob_backend;
|
||||
pub mod chunked_upload_service;
|
||||
pub mod compression_service;
|
||||
pub mod consistency_batch_service;
|
||||
pub mod db_pool_monitor;
|
||||
pub mod dedup_service;
|
||||
pub mod drives_consistency_service;
|
||||
pub mod encrypted_blob_backend;
|
||||
pub mod exif_service;
|
||||
pub mod face_geometry;
|
||||
@@ -12,6 +16,8 @@ pub mod face_indexing_service;
|
||||
pub mod ffmpeg_video_frame_service;
|
||||
pub mod file_content_cache;
|
||||
pub mod file_system_i18n_service;
|
||||
pub mod files_consistency_service;
|
||||
pub mod folders_consistency_service;
|
||||
pub mod grant_cleanup_service;
|
||||
pub mod image_transcode_service;
|
||||
pub mod jwt_service;
|
||||
|
||||
@@ -355,4 +355,26 @@ impl BlobStorageBackend for RetryBlobBackend {
|
||||
fn local_blob_path(&self, hash: &str) -> Option<PathBuf> {
|
||||
self.inner.local_blob_path(hash)
|
||||
}
|
||||
|
||||
/// Enumeration delegates to inner. Retry semantics apply per
|
||||
/// call, not per batch — a single list call that fails after
|
||||
/// exhausting retries surfaces the error to the tenant, which
|
||||
/// treats it as a transient backend issue and skips the batch.
|
||||
fn list_blob_hashes(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: usize,
|
||||
) -> Pin<
|
||||
Box<
|
||||
dyn std::future::Future<
|
||||
Output = Result<
|
||||
crate::application::ports::blob_storage_ports::BlobListPage,
|
||||
DomainError,
|
||||
>,
|
||||
> + Send
|
||||
+ '_,
|
||||
>,
|
||||
> {
|
||||
self.inner.list_blob_hashes(cursor, limit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -421,4 +421,97 @@ impl BlobStorageBackend for S3BlobBackend {
|
||||
fn local_blob_path(&self, _hash: &str) -> Option<PathBuf> {
|
||||
None // Remote backend — no local path
|
||||
}
|
||||
|
||||
/// Enumerate blobs via S3 `ListObjectsV2`. Cursor is the S3
|
||||
/// continuation token verbatim (opaque). Filter: keys must
|
||||
/// match `<xx>/<64-hex>.blob` — matches how `blob_key` writes
|
||||
/// them — so any future non-blob namespace living in the same
|
||||
/// bucket (e.g. `thumbnails/<hash>.jpg`) is skipped
|
||||
/// automatically. No prefix passed to S3 so we get everything
|
||||
/// in one paginated scan; the client-side filter enforces
|
||||
/// correctness.
|
||||
fn list_blob_hashes(
|
||||
&self,
|
||||
cursor: Option<String>,
|
||||
limit: usize,
|
||||
) -> Pin<
|
||||
Box<
|
||||
dyn std::future::Future<
|
||||
Output = Result<
|
||||
crate::application::ports::blob_storage_ports::BlobListPage,
|
||||
DomainError,
|
||||
>,
|
||||
> + Send
|
||||
+ '_,
|
||||
>,
|
||||
> {
|
||||
use crate::application::ports::blob_storage_ports::{
|
||||
BackendBlobEntry, BackendUnknownEntry, BlobListPage,
|
||||
};
|
||||
|
||||
Box::pin(async move {
|
||||
let mut req = self
|
||||
.client
|
||||
.list_objects_v2()
|
||||
.bucket(&self.bucket)
|
||||
.max_keys(limit.min(1000) as i32);
|
||||
if let Some(c) = cursor {
|
||||
req = req.continuation_token(c);
|
||||
}
|
||||
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
DomainError::new(
|
||||
ErrorKind::InternalError,
|
||||
"Blob",
|
||||
format!("S3 ListObjectsV2 failed: {e}"),
|
||||
)
|
||||
})?;
|
||||
|
||||
let objects = resp.contents.unwrap_or_default();
|
||||
let mut blobs: Vec<BackendBlobEntry> = Vec::with_capacity(objects.len());
|
||||
let mut unknowns: Vec<BackendUnknownEntry> = Vec::new();
|
||||
|
||||
for obj in objects {
|
||||
let Some(key) = obj.key else { continue };
|
||||
let mtime = obj.last_modified.and_then(|ts| {
|
||||
let secs = ts.secs();
|
||||
let nsecs = ts.subsec_nanos();
|
||||
chrono::DateTime::<chrono::Utc>::from_timestamp(secs, nsecs)
|
||||
});
|
||||
|
||||
// Canonical S3 key shape: `<xx>/<64-hex>.blob`.
|
||||
// Anything else is a sidecar or foreign namespace
|
||||
// (e.g. future `thumbnails/<hash>.jpg` if Ed adds
|
||||
// that) — surface as an unknown so operators know
|
||||
// it's there. Recovery framework can decide per-
|
||||
// pattern how to act.
|
||||
let is_canonical = key.split_once('/').and_then(|(prefix, rest)| {
|
||||
if prefix.len() != 2 || !prefix.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||
return None;
|
||||
}
|
||||
rest.strip_suffix(".blob")
|
||||
.filter(|stem| {
|
||||
stem.len() == 64 && stem.chars().all(|c| c.is_ascii_hexdigit())
|
||||
})
|
||||
.map(|s| s.to_string())
|
||||
});
|
||||
|
||||
match is_canonical {
|
||||
Some(hash) => blobs.push(BackendBlobEntry { hash, mtime }),
|
||||
None => unknowns.push(BackendUnknownEntry { path: key, mtime }),
|
||||
}
|
||||
}
|
||||
|
||||
let next_cursor = if resp.is_truncated.unwrap_or(false) {
|
||||
resp.next_continuation_token
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(BlobListPage {
|
||||
blobs,
|
||||
unknowns,
|
||||
next_cursor,
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use tracing::{debug, error, info, instrument};
|
||||
use crate::common::errors::Result;
|
||||
use crate::domain::repositories::trash_repository::TrashRepository;
|
||||
use crate::infrastructure::repositories::pg::trash_db_repository::TrashDbRepository;
|
||||
use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRunArgs};
|
||||
use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs};
|
||||
use crate::infrastructure::services::dedup_service::DedupService;
|
||||
use async_trait::async_trait;
|
||||
|
||||
@@ -43,12 +43,31 @@ impl TrashCleanupService {
|
||||
}
|
||||
}
|
||||
|
||||
/// Registered interval as a `Duration` — helper for DI wiring so
|
||||
/// the composition root doesn't reinvent the `hours × 3600` cast.
|
||||
/// Registered interval as a `Duration`. Internal helper used by
|
||||
/// [`Self::register`]; kept `pub` in case a test wants to assert
|
||||
/// the clamped value.
|
||||
pub fn interval(&self) -> Duration {
|
||||
Duration::from_secs(self.cleanup_interval_hours * 3600)
|
||||
}
|
||||
|
||||
/// Register self with the periodic-job scheduler and return the
|
||||
/// same `Arc<Self>` for DI-style method chaining:
|
||||
///
|
||||
/// ```ignore
|
||||
/// let svc = Arc::new(TrashCleanupService::new(...))
|
||||
/// .register(&core.job_registry)
|
||||
/// .await;
|
||||
/// ```
|
||||
///
|
||||
/// Scheduled tenant — interval reads from
|
||||
/// `self.cleanup_interval_hours`, no timeout. See
|
||||
/// `docs/plan/job-registry.md` Part 1 §Contract.
|
||||
pub async fn register(self: Arc<Self>, registry: &JobRegistry) -> Arc<Self> {
|
||||
let interval = self.interval();
|
||||
registry.register(self.clone(), Some(interval), None).await;
|
||||
self
|
||||
}
|
||||
|
||||
/// Starts the periodic cleanup job
|
||||
#[instrument(skip(self))]
|
||||
pub async fn start_cleanup_job(&self) {
|
||||
|
||||
@@ -147,8 +147,26 @@ pub fn admin_routes() -> Router<Arc<AppState>> {
|
||||
// audit-logged. See `docs/plan/job-registry.md` §Cross-cutting.
|
||||
// Retired the `/internal/trigger-sweep|gc|grant-cleanup` shims
|
||||
// that used to sit here (Stage 2 of the job-registry rollout).
|
||||
//
|
||||
// `/jobs` + `/jobs/{name}/trigger` cover every registered
|
||||
// JobHandler (periodic + recoverable — recoverable ones slot
|
||||
// in through `RecoverableAdapter`). The `/cancel` + `/runs`
|
||||
// + `/runs/{id}` triplet is recoverable-only — hitting them
|
||||
// on a stateless job silently gets an empty list / no-op
|
||||
// cancel, since no rows in `jobs.recoverable_runs` match.
|
||||
// See `docs/plan/job-registry.md` Part 2.
|
||||
.route("/jobs", get(list_jobs))
|
||||
.route("/jobs/{name}/trigger", post(trigger_job))
|
||||
.route("/jobs/{name}/cancel", post(cancel_job))
|
||||
.route("/jobs/{name}/runs", get(list_job_runs))
|
||||
.route("/jobs/{name}/runs/{id}", get(get_job_run))
|
||||
.route(
|
||||
"/jobs/{name}/runs/{id}/findings",
|
||||
get(list_job_run_findings),
|
||||
)
|
||||
// Retention cleanup — operator-triggered, not periodic.
|
||||
// See `purge_job_runs` docstring for the semantics.
|
||||
.route("/jobs/runs/purge", post(purge_job_runs))
|
||||
// Drives — admin-wide view (distinct from `/api/drives` which
|
||||
// is filtered to the caller's role grants).
|
||||
.route("/drives", get(list_all_drives))
|
||||
@@ -2084,10 +2102,16 @@ pub async fn list_jobs(State(state): State<Arc<AppState>>) -> impl IntoResponse
|
||||
/// support it (dedup_gc → grace = 0, grant_cleanup → grace = 0).
|
||||
/// Silently ignored by handlers that don't (trash_cleanup,
|
||||
/// storage_reconcile).
|
||||
///
|
||||
/// `deep=true` opts into slow variants — `consistency_batch` fans it
|
||||
/// out to sub-jobs; `storage_consistency` (when implemented) will
|
||||
/// re-BLAKE3 each blob for bitrot detection. See `JobRunArgs.deep`.
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct TriggerJobQuery {
|
||||
#[serde(default)]
|
||||
pub force: bool,
|
||||
#[serde(default)]
|
||||
pub deep: bool,
|
||||
}
|
||||
|
||||
/// `POST /api/admin/jobs/{name}/trigger` — dispatch one run off-schedule.
|
||||
@@ -2125,11 +2149,16 @@ pub async fn trigger_job(
|
||||
event = "job.trigger",
|
||||
job = %name,
|
||||
force = query.force,
|
||||
"👮🏻♂️ Admin triggered job {} (force={})",
|
||||
deep = query.deep,
|
||||
"👮🏻♂️ Admin triggered job {} (force={}, deep={})",
|
||||
name,
|
||||
query.force,
|
||||
query.deep,
|
||||
);
|
||||
let args = JobRunArgs { force: query.force };
|
||||
let args = JobRunArgs {
|
||||
force: query.force,
|
||||
deep: query.deep,
|
||||
};
|
||||
match state.core.job_registry.trigger(&name, &args).await {
|
||||
Some(outcome) => (
|
||||
StatusCode::OK,
|
||||
@@ -2146,3 +2175,294 @@ pub async fn trigger_job(
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST /api/admin/jobs/{name}/cancel` — cooperative cancel of the
|
||||
/// currently-running recoverable run for `{name}`.
|
||||
///
|
||||
/// Flips the row's `status` from `Running` → `CancelRequested`. The
|
||||
/// handler is responsible for polling `store.status()` between batches
|
||||
/// and returning `RunOutcome::Paused` at the next safe boundary; if it
|
||||
/// doesn't, the cancel is a no-op until the run completes naturally.
|
||||
///
|
||||
/// Returns 200 with the run_id when a Running row was flipped, 200 with
|
||||
/// `cancelled: false` when nothing was running (either no runs exist,
|
||||
/// or the latest is Paused / Completed / Failed / already CancelRequested).
|
||||
/// Never 404 on "no active run" — the job name is registered and the
|
||||
/// endpoint just reports the truth.
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/admin/jobs/{name}/cancel",
|
||||
params(("name" = String, Path, description = "Registered job name")),
|
||||
responses(
|
||||
(status = 200, description = "Cancel signalled (or no-op if nothing was running)"),
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 403, description = "Admin required"),
|
||||
(status = 500, description = "DB error"),
|
||||
),
|
||||
security(("bearerAuth" = [])),
|
||||
tag = "admin"
|
||||
)]
|
||||
pub async fn cancel_job(
|
||||
State(state): State<Arc<AppState>>,
|
||||
axum::extract::Path(name): axum::extract::Path<String>,
|
||||
) -> impl IntoResponse {
|
||||
use crate::infrastructure::scheduler::JobStoreProvider as _;
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "job.cancel_requested",
|
||||
job = %name,
|
||||
"👮🏻♂️ Admin requested cancel for job {}",
|
||||
name,
|
||||
);
|
||||
match state.core.job_store_provider.request_cancel(&name).await {
|
||||
Ok(Some(run_id)) => (
|
||||
StatusCode::OK,
|
||||
Json(serde_json::json!({
|
||||
"cancelled": true,
|
||||
"run_id": run_id.to_string(),
|
||||
"status": "CancelRequested",
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
Ok(None) => (
|
||||
StatusCode::OK,
|
||||
Json(serde_json::json!({
|
||||
"cancelled": false,
|
||||
"reason": "no running run for this job",
|
||||
})),
|
||||
)
|
||||
.into_response(),
|
||||
Err(e) => AppError::internal_error(format!("cancel failed: {e}")).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Query parameters for `GET /api/admin/jobs/{name}/runs`.
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct ListRunsQuery {
|
||||
/// Cap on returned rows. Server-side clamps to 100 defensively.
|
||||
#[serde(default = "default_runs_limit")]
|
||||
pub limit: u32,
|
||||
}
|
||||
|
||||
fn default_runs_limit() -> u32 {
|
||||
20
|
||||
}
|
||||
|
||||
/// `GET /api/admin/jobs/{name}/runs?limit=N` — history of recoverable
|
||||
/// runs for a registered job, newest first. Includes terminal +
|
||||
/// non-terminal rows.
|
||||
///
|
||||
/// Read-only, no audit line — standard admin-middleware auth is enough.
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/admin/jobs/{name}/runs",
|
||||
params(
|
||||
("name" = String, Path, description = "Registered job name"),
|
||||
("limit" = Option<u32>, Query, description = "Max rows to return (default 20, capped at 100)"),
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "Runs listed (may be empty)"),
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 403, description = "Admin required"),
|
||||
(status = 500, description = "DB error"),
|
||||
),
|
||||
security(("bearerAuth" = [])),
|
||||
tag = "admin"
|
||||
)]
|
||||
pub async fn list_job_runs(
|
||||
State(state): State<Arc<AppState>>,
|
||||
axum::extract::Path(name): axum::extract::Path<String>,
|
||||
axum::extract::Query(query): axum::extract::Query<ListRunsQuery>,
|
||||
) -> impl IntoResponse {
|
||||
use crate::infrastructure::scheduler::JobStoreProvider as _;
|
||||
let limit = query.limit.clamp(1, 100);
|
||||
match state.core.job_store_provider.list_runs(&name, limit).await {
|
||||
Ok(runs) => (StatusCode::OK, Json(runs)).into_response(),
|
||||
Err(e) => AppError::internal_error(format!("list_runs failed: {e}")).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /api/admin/jobs/{name}/runs/{id}` — single-run detail.
|
||||
///
|
||||
/// Returns 404 when the id doesn't exist. `{name}` is not validated
|
||||
/// against the run's `job_name` — the id is globally unique — but
|
||||
/// keeping the name in the URL path lets operators build stable
|
||||
/// per-job history links without knowing individual run ids upfront.
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/admin/jobs/{name}/runs/{id}",
|
||||
params(
|
||||
("name" = String, Path, description = "Registered job name"),
|
||||
("id" = String, Path, description = "Run UUID"),
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "Run detail"),
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 403, description = "Admin required"),
|
||||
(status = 404, description = "Run not found"),
|
||||
(status = 500, description = "DB error"),
|
||||
),
|
||||
security(("bearerAuth" = [])),
|
||||
tag = "admin"
|
||||
)]
|
||||
pub async fn get_job_run(
|
||||
State(state): State<Arc<AppState>>,
|
||||
axum::extract::Path((_name, id)): axum::extract::Path<(String, uuid::Uuid)>,
|
||||
) -> impl IntoResponse {
|
||||
use crate::infrastructure::scheduler::JobStoreProvider as _;
|
||||
match state.core.job_store_provider.get_run_by_id(id).await {
|
||||
Ok(Some(run)) => (StatusCode::OK, Json(run)).into_response(),
|
||||
Ok(None) => (
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(serde_json::json!({ "error": "run not found", "id": id.to_string() })),
|
||||
)
|
||||
.into_response(),
|
||||
Err(e) => AppError::internal_error(format!("get_run failed: {e}")).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Query parameters for `GET /api/admin/jobs/{name}/runs/{id}/findings`.
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct ListFindingsQuery {
|
||||
/// Page size — server clamps to 500 defensively.
|
||||
#[serde(default = "default_findings_limit")]
|
||||
pub limit: u32,
|
||||
#[serde(default)]
|
||||
pub offset: u32,
|
||||
}
|
||||
|
||||
fn default_findings_limit() -> u32 {
|
||||
100
|
||||
}
|
||||
|
||||
/// `GET /api/admin/jobs/{name}/runs/{id}/findings?limit=N&offset=M` —
|
||||
/// paginated findings emitted by a specific run of a recoverable job.
|
||||
///
|
||||
/// 404 when the run id doesn't exist (anti-enum: caller knew the id
|
||||
/// somehow; we don't leak whether it was pruned vs never-existed).
|
||||
/// Read-only, no audit — standard admin-middleware auth is enough.
|
||||
///
|
||||
/// `{name}` is not validated against the run's `job_name` (the id is
|
||||
/// globally unique) but keeps the URL path consistent with the other
|
||||
/// per-run endpoints for stable per-job history links.
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/admin/jobs/{name}/runs/{id}/findings",
|
||||
params(
|
||||
("name" = String, Path, description = "Registered job name"),
|
||||
("id" = String, Path, description = "Run UUID"),
|
||||
("limit" = Option<u32>, Query, description = "Max rows (default 100, capped at 500)"),
|
||||
("offset" = Option<u32>, Query, description = "Rows to skip (default 0)"),
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "Findings listed (may be empty)"),
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 403, description = "Admin required"),
|
||||
(status = 404, description = "Run not found"),
|
||||
(status = 500, description = "DB error"),
|
||||
),
|
||||
security(("bearerAuth" = [])),
|
||||
tag = "admin"
|
||||
)]
|
||||
pub async fn list_job_run_findings(
|
||||
State(state): State<Arc<AppState>>,
|
||||
axum::extract::Path((_name, id)): axum::extract::Path<(String, uuid::Uuid)>,
|
||||
axum::extract::Query(query): axum::extract::Query<ListFindingsQuery>,
|
||||
) -> impl IntoResponse {
|
||||
use crate::infrastructure::scheduler::JobStoreProvider as _;
|
||||
// Existence check first — otherwise a paged listing of a
|
||||
// nonexistent run returns 200 [] which is indistinguishable from
|
||||
// "run exists, no findings" and breaks operator drill-down.
|
||||
match state.core.job_store_provider.get_run_by_id(id).await {
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => {
|
||||
return (
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(serde_json::json!({ "error": "run not found", "id": id.to_string() })),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
Err(e) => return AppError::internal_error(format!("get_run failed: {e}")).into_response(),
|
||||
}
|
||||
let limit = query.limit.clamp(1, 500);
|
||||
match state
|
||||
.core
|
||||
.job_store_provider
|
||||
.list_findings(id, limit, query.offset)
|
||||
.await
|
||||
{
|
||||
Ok(findings) => (StatusCode::OK, Json(findings)).into_response(),
|
||||
Err(e) => AppError::internal_error(format!("list_findings failed: {e}")).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Query parameters for `POST /api/admin/jobs/runs/purge`.
|
||||
///
|
||||
/// `days` — retention window. Terminal runs (`Completed`, `Failed`)
|
||||
/// with `completed_at` older than this many days ago are deleted
|
||||
/// (with their findings via CASCADE). Default 30. Minimum enforced
|
||||
/// at 1 by the provider — zero would eat runs completed seconds
|
||||
/// ago. Non-terminal runs are ALWAYS preserved regardless of age.
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct PurgeJobRunsQuery {
|
||||
#[serde(default = "default_purge_days")]
|
||||
pub days: i32,
|
||||
}
|
||||
|
||||
fn default_purge_days() -> i32 {
|
||||
30
|
||||
}
|
||||
|
||||
/// `POST /api/admin/jobs/runs/purge?days=N` — operator-triggered
|
||||
/// cleanup of old terminal runs + their findings. Not periodic;
|
||||
/// admins fire this when they want to reclaim `jobs.*` history
|
||||
/// space. Delegates entirely to
|
||||
/// `JobStoreProvider::purge_terminal_runs` — no SQL in the handler
|
||||
/// (see `AGENTS.md` § handler thinness).
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/admin/jobs/runs/purge",
|
||||
params(
|
||||
("days" = Option<i32>, Query, description = "Retention window in days (default 30, minimum 1). Terminal runs older than this are deleted with their findings; non-terminal runs are always preserved."),
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "Purge complete"),
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 403, description = "Admin required"),
|
||||
(status = 500, description = "DB error"),
|
||||
),
|
||||
security(("bearerAuth" = [])),
|
||||
tag = "admin"
|
||||
)]
|
||||
pub async fn purge_job_runs(
|
||||
State(state): State<Arc<AppState>>,
|
||||
axum::extract::Query(query): axum::extract::Query<PurgeJobRunsQuery>,
|
||||
) -> impl IntoResponse {
|
||||
use crate::infrastructure::scheduler::JobStoreProvider as _;
|
||||
let retention_days = query.days.max(1);
|
||||
match state
|
||||
.core
|
||||
.job_store_provider
|
||||
.purge_terminal_runs(retention_days)
|
||||
.await
|
||||
{
|
||||
Ok(purged) => {
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "jobs.runs_purged",
|
||||
purged = purged,
|
||||
retention_days = retention_days,
|
||||
"👮🏻♂️ admin purged {purged} terminal recoverable-run row(s) past {retention_days} day retention (findings cascaded)",
|
||||
);
|
||||
(
|
||||
StatusCode::OK,
|
||||
Json(serde_json::json!({
|
||||
"purged": purged,
|
||||
"retention_days": retention_days,
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
Err(e) => AppError::internal_error(format!("purge failed: {e}")).into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,6 +234,9 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload;
|
||||
// (docs/plan/job-registry.md §Cross-cutting).
|
||||
handlers::admin_handler::list_jobs,
|
||||
handlers::admin_handler::trigger_job,
|
||||
handlers::admin_handler::cancel_job,
|
||||
handlers::admin_handler::list_job_runs,
|
||||
handlers::admin_handler::get_job_run,
|
||||
// Grant / ReBAC handlers (free functions)
|
||||
handlers::grant_handler::create_grant,
|
||||
handlers::grant_handler::revoke_grant,
|
||||
|
||||
+116
-1
@@ -89,7 +89,19 @@ jsonpath "$..interval_ms" contains 600000
|
||||
jsonpath "$..interval_ms" count == 3
|
||||
|
||||
# Every entry carries a `running` bool — same aggregate primitive.
|
||||
jsonpath "$..running" count == 4
|
||||
# Count matches the registered-tenant count: 4 Part 1 periodics
|
||||
# (trash_cleanup, storage_reconcile, dedup_gc, grant_cleanup) + 5
|
||||
# Part 2 recoverables (drives_consistency, folders_consistency,
|
||||
# files_consistency, blobs_consistency, backend_consistency —
|
||||
# wrapped by RecoverableAdapter so they appear here alongside the
|
||||
# periodics) + 1 coordinator (consistency_batch — a plain
|
||||
# JobHandler that dispatches every registered `*_consistency`).
|
||||
# Bump when a new tenant registers.
|
||||
jsonpath "$..running" count == 10
|
||||
jsonpath "$[*].name" contains "drives_consistency"
|
||||
jsonpath "$[*].name" contains "folders_consistency"
|
||||
jsonpath "$[*].name" contains "files_consistency"
|
||||
jsonpath "$[*].name" contains "consistency_batch"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
@@ -126,6 +138,109 @@ HTTP 200
|
||||
jsonpath "$..last_outcome.outcome" contains "ok"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 4b — Trigger `folders_consistency`. Second Part 2
|
||||
# recoverable tenant. Goes through the same
|
||||
# RecoverableAdapter → PgJobStoreProvider path as
|
||||
# drives_consistency (opens a run row, walks the
|
||||
# `storage.folders` cursor, marks Completed). Success
|
||||
# envelope shape identical.
|
||||
#
|
||||
# Captures the run_id so Step 4b-findings can pin the
|
||||
# `GET /runs/{id}/findings` endpoint.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/admin/jobs/folders_consistency/trigger
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
folders_run_id: jsonpath "$.outcome.extra.run_id"
|
||||
[Asserts]
|
||||
jsonpath "$.ok" == true
|
||||
jsonpath "$.outcome.outcome" == "ok"
|
||||
jsonpath "$.outcome.count" exists
|
||||
jsonpath "$.outcome.extra.completed" == true
|
||||
jsonpath "$.outcome.extra.run_id" exists
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 4b-findings — List findings for the run we just kicked.
|
||||
# The response body is a JSON array (possibly empty on a clean
|
||||
# test DB — the fresh Hurl DB has no folder-tree drift). Assert:
|
||||
# * 200 on a real run_id.
|
||||
# * response is an array (isCollection covers both empty + non-
|
||||
# empty). Structural shape of individual finding rows is pinned
|
||||
# by the drives_consistency_service integration test which seeds
|
||||
# drift and asserts kind/severity/detail — not repeated here.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/admin/jobs/folders_consistency/runs/{{folders_run_id}}/findings
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$" isCollection
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 4b-findings-404 — Findings for a run_id that doesn't
|
||||
# exist. Endpoint returns 404, not 200 [] — otherwise a broken
|
||||
# link from the admin UI would look like "no findings" instead
|
||||
# of "run missing".
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/admin/jobs/folders_consistency/runs/00000000-0000-0000-0000-000000000000/findings
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 404
|
||||
[Asserts]
|
||||
jsonpath "$.error" == "run not found"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 4b2 — Trigger `files_consistency`. Third Part 2 recoverable
|
||||
# tenant. Iterates `storage.files` and self-joins folder
|
||||
# + blob. Same envelope shape as the earlier consistency
|
||||
# tenants.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/admin/jobs/files_consistency/trigger
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.ok" == true
|
||||
jsonpath "$.outcome.outcome" == "ok"
|
||||
jsonpath "$.outcome.count" exists
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 4c — Trigger `consistency_batch`. Coordinator (plain
|
||||
# JobHandler) — snapshots the registry, filters names
|
||||
# ending `_consistency`, sequentially triggers each.
|
||||
# `outcome.count` = number of children dispatched (5 as
|
||||
# of Slice 10: drives + folders + files + blobs +
|
||||
# backend). `extra.per_check` carries a per-child outcome
|
||||
# map. Batch itself always returns ok — child failures
|
||||
# live inside per_check. `?deep=true` propagates as
|
||||
# `extra.deep`.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/admin/jobs/consistency_batch/trigger?deep=true
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.ok" == true
|
||||
jsonpath "$.outcome.outcome" == "ok"
|
||||
jsonpath "$.outcome.count" == 5
|
||||
jsonpath "$.outcome.extra.deep" == true
|
||||
jsonpath "$.outcome.extra.ok" == 5
|
||||
jsonpath "$.outcome.extra.err" == 0
|
||||
# per_check is keyed by child job name.
|
||||
jsonpath "$.outcome.extra.per_check.drives_consistency.outcome" == "ok"
|
||||
jsonpath "$.outcome.extra.per_check.folders_consistency.outcome" == "ok"
|
||||
jsonpath "$.outcome.extra.per_check.files_consistency.outcome" == "ok"
|
||||
jsonpath "$.outcome.extra.per_check.blobs_consistency.outcome" == "ok"
|
||||
jsonpath "$.outcome.extra.per_check.backend_consistency.outcome" == "ok"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 5 — Trigger a job that doesn't exist. 404 anti-enum on
|
||||
# `JobRegistry::trigger` returning `None`.
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
# =============================================================
|
||||
# OxiCloud — Recoverable-run admin surface
|
||||
# =============================================================
|
||||
# Pins the Part 2 (recoverable-run engine) admin endpoints:
|
||||
# * POST /api/admin/jobs/{name}/trigger (RecoverableJobHandler
|
||||
# path via RecoverableAdapter)
|
||||
# * POST /api/admin/jobs/{name}/cancel
|
||||
# * GET /api/admin/jobs/{name}/runs
|
||||
# * GET /api/admin/jobs/{name}/runs/{id}
|
||||
#
|
||||
# Uses `drives_consistency` — the first recoverable tenant, on-demand
|
||||
# only. Verifies:
|
||||
# 1. Registered job appears in `GET /api/admin/jobs` with no
|
||||
# interval (on-demand only).
|
||||
# 2. Triggering creates a fresh row in `jobs.recoverable_runs`,
|
||||
# handler completes, run terminates as Completed.
|
||||
# 3. History endpoint returns the just-completed run.
|
||||
# 4. Single-run detail endpoint returns the same row.
|
||||
# 5. Cancel-on-idle is a no-op with `cancelled: false` (nothing
|
||||
# running to cancel).
|
||||
# 6. Unknown run id → 404 on the single-run endpoint.
|
||||
# 7. Non-admin caller → 403 from the admin middleware on every
|
||||
# recoverable endpoint (no bespoke role check in the handlers).
|
||||
#
|
||||
# Drift-finding assertions land alongside the `jobs.run_findings`
|
||||
# migration — the current build LOGS findings to
|
||||
# `oxicloud::consistency` without persisting them. Log-tail
|
||||
# assertions from Hurl are fragile so we defer them.
|
||||
# =============================================================
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Setup — admin login + rjobs_bob (non-admin) provisioning
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/auth/login
|
||||
Content-Type: application/json
|
||||
{ "username": "{{username}}", "password": "{{password}}" }
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
admin_token: jsonpath "$.access_token"
|
||||
|
||||
|
||||
# Anti-enum registration.
|
||||
POST {{base_url}}/api/auth/register
|
||||
Content-Type: application/json
|
||||
{
|
||||
"username": "rjobs_bob",
|
||||
"email": "rjobs_bob@example.com",
|
||||
"password": "RjobsBobPassword1!"
|
||||
}
|
||||
|
||||
HTTP 200
|
||||
|
||||
|
||||
POST {{base_url}}/api/auth/login
|
||||
Content-Type: application/json
|
||||
{ "username": "rjobs_bob", "password": "RjobsBobPassword1!" }
|
||||
|
||||
HTTP 200
|
||||
[Captures]
|
||||
bob_token: jsonpath "$.access_token"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 1 — `drives_consistency` is registered on-demand only.
|
||||
# Appears in the listing without an `interval_ms`.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/admin/jobs
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$[*].name" contains "drives_consistency"
|
||||
# On-demand → no interval_ms (`skip_serializing_if = Option::is_none`).
|
||||
jsonpath "$[?(@.name=='drives_consistency')].interval_ms" not exists
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 2 — Trigger the check. Handler dispatches through
|
||||
# `RecoverableAdapter` → `run_or_resume`, which INSERTs
|
||||
# a fresh `jobs.recoverable_runs` row, runs the scan,
|
||||
# marks it Completed. Response envelope:
|
||||
# { ok, outcome: { outcome: "ok",
|
||||
# extra: { completed: true, run_id: "..." } } }
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/admin/jobs/drives_consistency/trigger
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.ok" == true
|
||||
jsonpath "$.outcome.outcome" == "ok"
|
||||
jsonpath "$.outcome.extra.completed" == true
|
||||
[Captures]
|
||||
run_id: jsonpath "$.outcome.extra.run_id"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 3 — Run history returns at least the just-triggered
|
||||
# run, newest first. Response is a JSON array of
|
||||
# RunSummary; the top entry must be the run_id we
|
||||
# captured above with status='Completed'.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/admin/jobs/drives_consistency/runs
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$" isCollection
|
||||
jsonpath "$[0].id" == "{{run_id}}"
|
||||
jsonpath "$[0].job_name" == "drives_consistency"
|
||||
jsonpath "$[0].status" == "Completed"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 4 — Single-run detail. Returns the same row shape as
|
||||
# the listing but for one id.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/admin/jobs/drives_consistency/runs/{{run_id}}
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.id" == "{{run_id}}"
|
||||
jsonpath "$.job_name" == "drives_consistency"
|
||||
jsonpath "$.status" == "Completed"
|
||||
# `stats` is always present (JSONB NOT NULL DEFAULT '{}'); its
|
||||
# per-key shape is job-specific. `scanned_count` is bumped by the
|
||||
# handler's `checkpoint()` call, but the drives handler only
|
||||
# checkpoints when it processes a batch — a run that finds zero
|
||||
# rows in the first batch (e.g. all fixture drives sit inside the
|
||||
# 1h grace window on `storage.drives.created_at`) completes
|
||||
# without ever calling checkpoint, so `scanned_count` may be
|
||||
# absent. Pin `stats` existence; leave the counter unpinned.
|
||||
jsonpath "$.stats" exists
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 5 — Cancel-on-idle is a no-op. No Running row means no
|
||||
# Running→CancelRequested flip. Response is 200 with
|
||||
# `cancelled: false` (NOT a 404 — the job name is
|
||||
# registered, cancel just found nothing to cancel).
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/admin/jobs/drives_consistency/cancel
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 200
|
||||
[Asserts]
|
||||
jsonpath "$.cancelled" == false
|
||||
jsonpath "$.reason" == "no running run for this job"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 6 — Unknown run id → 404. UUID shape is valid; the id
|
||||
# just isn't in `jobs.recoverable_runs`.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
GET {{base_url}}/api/admin/jobs/drives_consistency/runs/00000000-0000-0000-0000-000000000000
|
||||
Authorization: Bearer {{admin_token}}
|
||||
|
||||
HTTP 404
|
||||
[Asserts]
|
||||
jsonpath "$.error" == "run not found"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# Step 7 — Non-admin caller is denied on every endpoint by
|
||||
# the `/api/admin/*` middleware layer. Handlers have
|
||||
# no bespoke role check — reaching them at all means
|
||||
# the caller is admin.
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
POST {{base_url}}/api/admin/jobs/drives_consistency/trigger
|
||||
Authorization: Bearer {{bob_token}}
|
||||
|
||||
HTTP 403
|
||||
|
||||
|
||||
POST {{base_url}}/api/admin/jobs/drives_consistency/cancel
|
||||
Authorization: Bearer {{bob_token}}
|
||||
|
||||
HTTP 403
|
||||
|
||||
|
||||
GET {{base_url}}/api/admin/jobs/drives_consistency/runs
|
||||
Authorization: Bearer {{bob_token}}
|
||||
|
||||
HTTP 403
|
||||
|
||||
|
||||
GET {{base_url}}/api/admin/jobs/drives_consistency/runs/{{run_id}}
|
||||
Authorization: Bearer {{bob_token}}
|
||||
|
||||
HTTP 403
|
||||
@@ -167,6 +167,7 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test
|
||||
"$API_DIR/dedup_blob_cleanup.hurl" \
|
||||
"$API_DIR/dedup_admin_gate.hurl" \
|
||||
"$API_DIR/admin_jobs.hurl" \
|
||||
"$API_DIR/recoverable_jobs.hurl" \
|
||||
"$API_DIR/default_caldav_carddav.hurl" \
|
||||
"$API_DIR/dav_error_mapping.hurl" \
|
||||
"$API_DIR/carddav_vcard_properties.hurl" \
|
||||
|
||||
+31
-39
@@ -15,44 +15,47 @@ test.beforeEach(async ({ page }) => {
|
||||
|
||||
test('walk every admin tab', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await expect(page.getByTestId('admin-dashboard-tab')).toBeVisible({ timeout: 15_000 });
|
||||
// Dashboard is the default tab.
|
||||
// Dashboard is the default section on `/admin` (bare path).
|
||||
// Assert on the sidebar entry (now the source of admin navigation)
|
||||
// and on content that only renders when Dashboard is active.
|
||||
await expect(page.getByTestId('appshell-nav-admin-dashboard-link')).toBeVisible({
|
||||
timeout: 15_000
|
||||
});
|
||||
await expect(page.getByTestId('admin-dashboard-registration-checkbox')).toBeVisible();
|
||||
|
||||
await page.getByTestId('admin-users-tab').click();
|
||||
await page.goto('/admin/users');
|
||||
await expect(page.getByTestId('admin-users-create-btn')).toBeVisible();
|
||||
|
||||
await page.getByTestId('admin-oidc-tab').click();
|
||||
await page.goto('/admin/oidc');
|
||||
await expect(page.getByTestId('admin-oidc-form')).toBeVisible();
|
||||
|
||||
await page.getByTestId('admin-storage-tab').click();
|
||||
await page.goto('/admin/storage');
|
||||
await expect(page.getByTestId('admin-storage-form')).toBeVisible();
|
||||
|
||||
await page.getByTestId('admin-smtp-tab').click();
|
||||
await page.goto('/admin/smtp');
|
||||
await expect(page.getByTestId('admin-smtp-send-btn')).toBeVisible();
|
||||
|
||||
await page.getByTestId('admin-plugins-tab').click();
|
||||
// Plugins panel content is conditional; assert the tab became active.
|
||||
await expect(page.getByTestId('admin-plugins-tab')).toHaveAttribute('aria-selected', 'true');
|
||||
await page.goto('/admin/plugins');
|
||||
// Plugins panel content is conditional; assert the URL landed
|
||||
// on the plugins section (proves routing worked; no content
|
||||
// guarantee).
|
||||
await expect(page).toHaveURL(/\/admin\/plugins$/);
|
||||
});
|
||||
|
||||
test('open the create-user form', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-users-tab').click();
|
||||
await page.goto('/admin/users');
|
||||
await page.getByTestId('admin-users-create-btn').click();
|
||||
await expect(page.getByTestId('admin-create-user-form')).toBeVisible({ timeout: 15_000 });
|
||||
});
|
||||
|
||||
test('storage tab: change backend select', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-storage-tab').click();
|
||||
await page.goto('/admin/storage');
|
||||
await expect(page.getByTestId('admin-storage-form')).toBeVisible({ timeout: 15_000 });
|
||||
await page.getByTestId('admin-storage-backend-select').selectOption({ index: 1 }).catch(() => {});
|
||||
});
|
||||
|
||||
test('storage tab: save the local backend settings', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-storage-tab').click();
|
||||
await page.goto('/admin/storage');
|
||||
await expect(page.getByTestId('admin-storage-form')).toBeVisible({ timeout: 15_000 });
|
||||
// Keep the (safe) local backend and save — exercises the save handler without
|
||||
// reconfiguring storage to a remote backend.
|
||||
@@ -62,8 +65,7 @@ test('storage tab: save the local backend settings', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('oidc tab: toggle enabled and fill issuer', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-oidc-tab').click();
|
||||
await page.goto('/admin/oidc');
|
||||
await expect(page.getByTestId('admin-oidc-form')).toBeVisible({ timeout: 15_000 });
|
||||
await page.getByTestId('admin-oidc-enabled-checkbox').check().catch(() => {});
|
||||
await page
|
||||
@@ -84,8 +86,7 @@ async function createUserRow(
|
||||
page: import('@playwright/test').Page,
|
||||
uname: string,
|
||||
): Promise<ReturnType<import('@playwright/test').Page['locator']>> {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-users-tab').click();
|
||||
await page.goto('/admin/users');
|
||||
await page.getByTestId('admin-users-create-btn').click();
|
||||
await expect(page.getByTestId('admin-create-user-form')).toBeVisible({ timeout: 15_000 });
|
||||
await page.getByTestId('admin-create-user-username-input').fill(uname);
|
||||
@@ -165,8 +166,7 @@ test('save a user quota and deactivate the user', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('save oidc settings', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-oidc-tab').click();
|
||||
await page.goto('/admin/oidc');
|
||||
await expect(page.getByTestId('admin-oidc-form')).toBeVisible({ timeout: 15_000 });
|
||||
await page.getByTestId('admin-oidc-issuer-input').fill('https://example.test/issuer').catch(() => {});
|
||||
await page.getByTestId('admin-oidc-client-id-input').fill('client-123').catch(() => {});
|
||||
@@ -175,9 +175,8 @@ test('save oidc settings', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('install, toggle, and delete a plugin', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-plugins-tab').click();
|
||||
await expect(page.getByTestId('admin-plugins-tab')).toHaveAttribute('aria-selected', 'true');
|
||||
await page.goto('/admin/plugins');
|
||||
await expect(page).toHaveURL(/\/admin\/plugins$/);
|
||||
|
||||
// Install the example hello plugin (plugins are enabled in the coverage env).
|
||||
await page.getByTestId('admin-plugins-install-input').setInputFiles(PLUGIN_ZIP);
|
||||
@@ -200,8 +199,7 @@ test('install, toggle, and delete a plugin', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('view plugin logs and details', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-plugins-tab').click();
|
||||
await page.goto('/admin/plugins');
|
||||
await page.getByTestId('admin-plugins-install-input').setInputFiles(PLUGIN_ZIP);
|
||||
await expect(page.locator('[data-testid^="admin-plugin-details-"]').first()).toBeVisible({
|
||||
timeout: 20_000,
|
||||
@@ -223,9 +221,8 @@ test('view plugin logs and details', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('plugins tab: save retention settings', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-plugins-tab').click();
|
||||
await expect(page.getByTestId('admin-plugins-tab')).toHaveAttribute('aria-selected', 'true');
|
||||
await page.goto('/admin/plugins');
|
||||
await expect(page).toHaveURL(/\/admin\/plugins$/);
|
||||
// The retention form is conditional; fill + save it when present.
|
||||
const retention = page.getByTestId('admin-plugin-retention-form');
|
||||
if (await retention.isVisible().catch(() => false)) {
|
||||
@@ -243,8 +240,7 @@ test('toggle the dashboard registration setting', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('send a test email from the smtp tab', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-smtp-tab').click();
|
||||
await page.goto('/admin/smtp');
|
||||
await expect(page.getByTestId('admin-smtp-to-input')).toBeVisible({ timeout: 15_000 });
|
||||
await page.getByTestId('admin-smtp-to-input').fill('test@example.test');
|
||||
await page.getByTestId('admin-smtp-send-btn').click();
|
||||
@@ -252,8 +248,7 @@ test('send a test email from the smtp tab', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('storage tab: fill the S3 backend fields', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-storage-tab').click();
|
||||
await page.goto('/admin/storage');
|
||||
await expect(page.getByTestId('admin-storage-form')).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
// Switch to S3 to reveal + fill the conditional fields (no save — that would
|
||||
@@ -270,8 +265,7 @@ test('storage tab: fill the S3 backend fields', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('oidc tab: run discovery against a bogus issuer', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-oidc-tab').click();
|
||||
await page.goto('/admin/oidc');
|
||||
await expect(page.getByTestId('admin-oidc-form')).toBeVisible({ timeout: 15_000 });
|
||||
await page.getByTestId('admin-oidc-issuer-input').fill('https://idp.example.test').catch(() => {});
|
||||
// Discovery fails (no real IdP) — exercises the discover + error path.
|
||||
@@ -286,8 +280,7 @@ test('users tab: paginate the user list', async ({ page }) => {
|
||||
for (let i = 0; i < 26; i++) {
|
||||
await apiAdminCreateUser(page, `pageu${Date.now()}${i}`);
|
||||
}
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-users-tab').click();
|
||||
await page.goto('/admin/users');
|
||||
await expect(page.getByTestId('admin-users-pager-next-btn')).toBeVisible({ timeout: 15_000 });
|
||||
await page.getByTestId('admin-users-pager-next-btn').click();
|
||||
await page.waitForTimeout(400);
|
||||
@@ -295,8 +288,7 @@ test('users tab: paginate the user list', async ({ page }) => {
|
||||
});
|
||||
|
||||
test('plugins tab: install then save retention settings', async ({ page }) => {
|
||||
await page.goto('/admin');
|
||||
await page.getByTestId('admin-plugins-tab').click();
|
||||
await page.goto('/admin/plugins');
|
||||
await page.getByTestId('admin-plugins-install-input').setInputFiles(PLUGIN_ZIP);
|
||||
await expect(page.locator('[data-testid^="admin-plugin-delete-"]').first()).toBeVisible({
|
||||
timeout: 20_000,
|
||||
|
||||
Reference in New Issue
Block a user