From 9902a6f8feb7a65e6bd0b97411f24fe6dd5917ba Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sun, 2 Aug 2026 13:36:20 +0200 Subject: [PATCH] refactor(usage_reconcile): explicit naming to prevent confusion with storage (backend) --- docs/architecture/jobs.md | 2 +- docs/plan/job-registry.md | 4 ++-- .../services/storage_usage_service.rs | 6 +++--- src/infrastructure/scheduler/types.rs | 2 +- .../services/drives_consistency_service.rs | 4 ++-- src/interfaces/api/handlers/admin_handler.rs | 2 +- tests/api/admin_jobs.hurl | 8 ++++---- tests/api/cross_drive_copy.hurl | 10 +++++----- tests/api/cross_drive_move.hurl | 10 +++++----- tests/api/drive_quota.hurl | 20 +++++++++---------- tests/api/storage_cleanup_check.sh | 6 +++--- tests/api/user_envelope_quota.hurl | 10 +++++----- 12 files changed, 42 insertions(+), 42 deletions(-) diff --git a/docs/architecture/jobs.md b/docs/architecture/jobs.md index 5195237e..f045d1c6 100644 --- a/docs/architecture/jobs.md +++ b/docs/architecture/jobs.md @@ -41,7 +41,7 @@ operator benefit. | Job name | Cadence | Force semantic | Service | |---|---|---|---| | `trash_cleanup` | 24 h (hardcoded in DI, no env var yet) | ignored | [`trash_cleanup_service.rs`](../../src/infrastructure/services/trash_cleanup_service.rs) | -| `storage_reconcile`| `OXICLOUD_STORAGE_USAGE_RECONCILE_SECS` (default 600s, min 30s) | ignored | [`storage_usage_service.rs`](../../src/application/services/storage_usage_service.rs) | +| `usage_reconcile` | `OXICLOUD_STORAGE_USAGE_RECONCILE_SECS` (default 600s, min 30s) | ignored | [`storage_usage_service.rs`](../../src/application/services/storage_usage_service.rs) | | `dedup_gc` | on-demand only (trash cleanup runs it inline as its tail step) | `force=true` → `garbage_collect_force()` (skip orphan grace) | [`dedup_service.rs`](../../src/infrastructure/services/dedup_service.rs) | | `grant_cleanup` | `OXICLOUD_GRANT_CLEANUP_INTERVAL_HOURS` (default 24h) — feature-gated by `OXICLOUD_GRANT_CLEANUP_ENABLED` | `force=true` → `purge(Some(0))` (grace_days=0) | [`grant_cleanup_service.rs`](../../src/infrastructure/services/grant_cleanup_service.rs) | diff --git a/docs/plan/job-registry.md b/docs/plan/job-registry.md index 0a82658e..44b87126 100644 --- a/docs/plan/job-registry.md +++ b/docs/plan/job-registry.md @@ -787,7 +787,7 @@ reference for any external tool that still expects the old paths: | Legacy (retired) | Replacement | |---|---| -| `POST /admin/internal/trigger-sweep` | `POST /admin/jobs/storage_reconcile/trigger` | +| `POST /admin/internal/trigger-sweep` | `POST /admin/jobs/usage_reconcile/trigger` | | `POST /admin/internal/trigger-gc?force=X` | `POST /admin/jobs/dedup_gc/trigger?force=X` | | `POST /admin/internal/trigger-grant-cleanup?force=X` | `POST /admin/jobs/grant_cleanup/trigger?force=X` | @@ -816,7 +816,7 @@ complete. Rough shape: │ Name Cadence Last run Status Actions │ │ ───────────────────────────────────────────────────────────────────│ │ trash_cleanup every 24 h 3h ago ok [Run] │ -│ storage_reconcile every 10 m 4m ago ok [Run] │ +│ usage_reconcile every 10 m 4m ago ok [Run] │ │ dedup_gc on-demand 1d ago ok [Run] │ │ grant_cleanup every 24 h never — [Run] │ │ drives_consistency on-demand never — [Run] │ diff --git a/src/application/services/storage_usage_service.rs b/src/application/services/storage_usage_service.rs index 2ce7cf0c..6ef66d36 100644 --- a/src/application/services/storage_usage_service.rs +++ b/src/application/services/storage_usage_service.rs @@ -65,7 +65,7 @@ impl StorageUsageService { /// `GET /api/drives` therefore lags by up to the cache TTL (30 s), /// which matches the sibling caches' accepted UX phantom for /// drive-name staleness. Tests / operators that need immediate - /// freshness call `POST /api/admin/jobs/storage_reconcile/trigger`, + /// freshness call `POST /api/admin/jobs/usage_reconcile/trigger`, /// which runs `update_all_drives_storage_usage` → this method. /// /// Security posture unaffected: `check_drive_quota` reads @@ -576,7 +576,7 @@ impl StorageUsageService { } } -pub const STORAGE_RECONCILE_JOB_NAME: &str = "storage_reconcile"; +pub const USAGE_RECONCILE_JOB_NAME: &str = "usage_reconcile"; use crate::infrastructure::scheduler::{JobHandler, JobOutcome, JobRegistry, JobRunArgs}; use async_trait::async_trait; @@ -600,7 +600,7 @@ impl StorageUsageService { #[async_trait] impl JobHandler for StorageUsageService { fn name(&self) -> &str { - STORAGE_RECONCILE_JOB_NAME + USAGE_RECONCILE_JOB_NAME } /// Runs both reconciliation sweeps — drives first, then users — diff --git a/src/infrastructure/scheduler/types.rs b/src/infrastructure/scheduler/types.rs index 7a3c8a9a..a2eac326 100644 --- a/src/infrastructure/scheduler/types.rs +++ b/src/infrastructure/scheduler/types.rs @@ -24,7 +24,7 @@ use serde::{Deserialize, Serialize}; /// Semantics of `force`, per job: /// - `dedup_gc` — skip the orphan grace window (grace = 0). /// - `grant_cleanup` — grace = 0. -/// - Others (trash_cleanup, storage_reconcile, …) — ignored. +/// - Others (trash_cleanup, usage_reconcile, …) — ignored. /// /// Semantics of `deep`, per job: /// - `consistency_batch` — propagate to sub-jobs; only `storage_consistency` diff --git a/src/infrastructure/services/drives_consistency_service.rs b/src/infrastructure/services/drives_consistency_service.rs index aad2d429..ea9da111 100644 --- a/src/infrastructure/services/drives_consistency_service.rs +++ b/src/infrastructure/services/drives_consistency_service.rs @@ -6,7 +6,7 @@ //! writes never decrement on `move_to_trash`) and the sweep at //! `storage_usage_service.rs::update_all_drives_storage_usage`. //! **Read-only** — reports drift as findings but does -//! NOT fix it. The existing `storage_reconcile` job (Part 1) is what +//! NOT fix it. The existing `usage_reconcile` job (Part 1) is what //! corrects the counter; this check surfaces WHEN drift happens so //! operators can trace it back to root cause (missed delta call, //! delta failed silently, race, etc.). @@ -145,7 +145,7 @@ impl RecoverableJobHandler for DrivesConsistencyCheck { // Fetch next batch of drives + their actual SUM in one // query. LEFT JOIN via correlated subquery gets us both - // sides in one round-trip; the storage_reconcile sweep + // sides in one round-trip; the usage_reconcile sweep // uses the same shape. // Grace window: skip drives created within the last hour. // A drive being created RIGHT NOW may still have its first diff --git a/src/interfaces/api/handlers/admin_handler.rs b/src/interfaces/api/handlers/admin_handler.rs index dd18e03a..8967c6ca 100644 --- a/src/interfaces/api/handlers/admin_handler.rs +++ b/src/interfaces/api/handlers/admin_handler.rs @@ -2335,7 +2335,7 @@ pub async fn list_jobs(State(state): State>) -> impl IntoResponse /// `force=true` requests acceleration semantics from handlers that /// support it (dedup_gc → grace = 0, grant_cleanup → grace = 0). /// Silently ignored by handlers that don't (trash_cleanup, -/// storage_reconcile). +/// usage_reconcile). /// /// `deep=true` opts into slow variants — `consistency_batch` fans it /// out to sub-jobs; `storage_consistency` (when implemented) will diff --git a/tests/api/admin_jobs.hurl b/tests/api/admin_jobs.hurl index 78f7a850..eeaa1b66 100644 --- a/tests/api/admin_jobs.hurl +++ b/tests/api/admin_jobs.hurl @@ -7,7 +7,7 @@ # # Coverage: # * Listing returns the four registered tenants -# (trash_cleanup, storage_reconcile, dedup_gc, grant_cleanup). +# (trash_cleanup, usage_reconcile, dedup_gc, grant_cleanup). # * Scheduled jobs report `interval_ms`; on-demand jobs # (`dedup_gc`) omit it via `skip_serializing_if=None`. # * Triggering a job updates its `last_outcome` in the next list. @@ -52,7 +52,7 @@ bob_token: jsonpath "$.access_token" # ───────────────────────────────────────────────────────────── # Step 1 — Admin lists jobs. All four Part 1 tenants must appear. -# Scheduled jobs (trash_cleanup, storage_reconcile, grant_cleanup) +# Scheduled jobs (trash_cleanup, usage_reconcile, grant_cleanup) # report `interval_ms`; on-demand jobs (`dedup_gc`) omit it via # serde's `skip_serializing_if = "Option::is_none"`. # @@ -69,7 +69,7 @@ HTTP 200 [Asserts] # Names — the four scheduler tenants. jsonpath "$[*].name" contains "trash_cleanup" -jsonpath "$[*].name" contains "storage_reconcile" +jsonpath "$[*].name" contains "usage_reconcile" jsonpath "$[*].name" contains "dedup_gc" jsonpath "$[*].name" contains "grant_cleanup" @@ -90,7 +90,7 @@ jsonpath "$..interval_ms" count == 3 # Every entry carries a `running` bool — same aggregate primitive. # Count matches the registered-tenant count: 4 Part 1 periodics -# (trash_cleanup, storage_reconcile, dedup_gc, grant_cleanup) + 5 +# (trash_cleanup, usage_reconcile, dedup_gc, grant_cleanup) + 5 # Part 2 recoverables (drives_consistency, folders_consistency, # files_consistency, blobs_consistency, backend_consistency — # wrapped by RecoverableAdapter so they appear here alongside the diff --git a/tests/api/cross_drive_copy.hurl b/tests/api/cross_drive_copy.hurl index 7dec25b1..b317705a 100644 --- a/tests/api/cross_drive_copy.hurl +++ b/tests/api/cross_drive_copy.hurl @@ -30,7 +30,7 @@ # same `v_dest_drive_id` variable, so (a) passing implies # file rows used the same value and (b) cross-checks it. # -# Sweep convergence: `/api/admin/jobs/storage_reconcile/trigger` is the +# Sweep convergence: `/api/admin/jobs/usage_reconcile/trigger` is the # deterministic synchronisation point — without it the # fire-and-forget delta hook may not yet have updated the cached # `used_bytes` when we read it. @@ -136,7 +136,7 @@ file_id: jsonpath "$.id" # numbers, the late hook adds its delta on top, and used_bytes ends # up high by exactly one file's size. Symptom: expected 32, got 64. # Real fix is await'ing the hook inline server-side. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -171,7 +171,7 @@ HTTP 200 [Captures] shared_file_id: jsonpath "$.successful[0].id" -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -259,7 +259,7 @@ HTTP 201 # the file's size into the cached counter. 200 ms is well above # the tokio task latency on any reasonable box; the deterministic # fix would be intra-transaction hooks, deferred until D7. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -348,7 +348,7 @@ jsonpath "$.name" == "dc-subtree-inner" # the Step 6 file copy (32) = 96. Anything other than (96, 96) # would mean the file INSERT in copy_folder_tree used the wrong # drive_id. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms diff --git a/tests/api/cross_drive_move.hurl b/tests/api/cross_drive_move.hurl index f6cbff6a..c23b4eca 100644 --- a/tests/api/cross_drive_move.hurl +++ b/tests/api/cross_drive_move.hurl @@ -17,7 +17,7 @@ # a file inside and watching the destination drive's # `used_bytes` jump by the descendant's size (not 0). # -# Sweep convergence: `/api/admin/jobs/storage_reconcile/trigger` is the +# Sweep convergence: `/api/admin/jobs/usage_reconcile/trigger` is the # deterministic synchronisation point — it recomputes every # drive's cached `used_bytes` from `SUM(file.size) WHERE # drive_id = d.id`. If the file/folder move didn't update @@ -125,7 +125,7 @@ file_id: jsonpath "$.id" # Baseline used_bytes after the upload settles. Trigger-sweep is # the deterministic sync point — but only after the spawn'd hook # has had a chance to land (bug_trigger_sweep_vs_spawn_hook_race.md). -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -161,7 +161,7 @@ Content-Type: application/json HTTP 200 -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -245,7 +245,7 @@ nested_file_id: jsonpath "$.id" # size. Symptom: expected 64, got 96 (one extra hook landed late). # Real fix is await'ing the hook inline server-side; until then this # delay deflakes the test. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -278,7 +278,7 @@ HTTP 200 # shared: nested hello-copy.txt now charged here (32) # Anything other than (32, 32) means the descendant file's # drive_id wasn't cascaded by the trigger. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms diff --git a/tests/api/drive_quota.hurl b/tests/api/drive_quota.hurl index a610ba99..e15e5ee1 100644 --- a/tests/api/drive_quota.hurl +++ b/tests/api/drive_quota.hurl @@ -120,7 +120,7 @@ small_file_id: jsonpath "$.id" # Ed's 2026-07-17 design call: the sweep is the escape hatch # for tests / operators that need immediate cache freshness; # per-write invalidation would nuke the cache on every upload. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -157,7 +157,7 @@ HTTP 201 # `used_bytes` climbs to 64 (32 + 32). Same trigger-sweep pattern # as the first assertion — the delta is fire-and-forget and the # listing cache lags until the sweep invalidates it. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -194,7 +194,7 @@ HTTP 507 # consumed by the intervening GET which re-populated the cache # with the pre-refused-write value. Sweep + re-check for # determinism. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} HTTP 200 @@ -238,7 +238,7 @@ HTTP 201 # Unlimited drive's `used_bytes` climbs to the file's exact size # (5 MiB = 5_242_880 bytes). Trigger-sweep pattern (see above). -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -266,7 +266,7 @@ jsonpath "$[?(@.id=='{{unlimited_drive_id}}')].used_bytes" == 5242880 # tight drive; the 5 MiB is in the unlimited one). # b) Permanently delete via empty-trash. # c) Trigger the reconciliation sweep on demand — -# `POST /api/admin/jobs/storage_reconcile/trigger`. +# `POST /api/admin/jobs/usage_reconcile/trigger`. # d) `GET /api/drives` now shows the corrected counter. # ───────────────────────────────────────────────────────────── DELETE {{base_url}}/api/files/{{small_file_id}} @@ -284,7 +284,7 @@ HTTP 200 # Sweep is fire-and-forget on a ticker (default 600 s). Run it now # so the assertion below is deterministic instead of polling. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} HTTP 200 @@ -310,7 +310,7 @@ jsonpath "$[?(@.id=='{{unlimited_drive_id}}')].used_bytes" == 5242880 # the route in production configs; here we just confirm a # non-admin caller is refused even when the feature is on. # ───────────────────────────────────────────────────────────── -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{owner_token}} HTTP 403 @@ -415,7 +415,7 @@ HTTP 200 # operations above never wrote anything. Trigger-sweep so the # check reads live SQL (see the class doc on the earlier # sweep + GET pair for the design rationale). -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} HTTP 200 @@ -570,7 +570,7 @@ HTTP 201 soft_shrink_file_id: jsonpath "$.id" -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -654,7 +654,7 @@ Authorization: Bearer {{owner_token}} HTTP 200 -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} HTTP 200 diff --git a/tests/api/storage_cleanup_check.sh b/tests/api/storage_cleanup_check.sh index 64d23aa3..36f55219 100755 --- a/tests/api/storage_cleanup_check.sh +++ b/tests/api/storage_cleanup_check.sh @@ -268,7 +268,7 @@ log "API confirms trash is empty." # disk state to be quiescent NOW. The two JobRegistry admin triggers # below (production surface, always on) make this deterministic: # -# 1. storage_reconcile — reconciles users.storage_used_bytes and +# 1. usage_reconcile — reconciles users.storage_used_bytes and # drives.used_bytes from SUM(size) — keeps # the cached counters honest for any quota # assertions that follow. @@ -280,8 +280,8 @@ log "API confirms trash is empty." # row-delete → unlink window the grace # normally protects. -curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/storage_reconcile/trigger" >/dev/null \ - || fail "storage_reconcile trigger failed" +curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/usage_reconcile/trigger" >/dev/null \ + || fail "usage_reconcile trigger failed" log "Reconciliation sweep triggered." GC_RESULT=$(curl -sf -X POST -H "$AUTH" "$base_url/api/admin/jobs/dedup_gc/trigger?force=true") diff --git a/tests/api/user_envelope_quota.hurl b/tests/api/user_envelope_quota.hurl index b1971c4a..ea141db0 100644 --- a/tests/api/user_envelope_quota.hurl +++ b/tests/api/user_envelope_quota.hurl @@ -19,7 +19,7 @@ # 4. Sweep self-heals — after trashing the personal file and # `trigger-sweep`, `/me.storage_used_bytes` returns to 0. # -# `POST /api/admin/jobs/storage_reconcile/trigger` is the +# `POST /api/admin/jobs/usage_reconcile/trigger` is the # deterministic synchronisation point: it runs the drive-side sweep # then the user-side sweep (both under the periodic scheduler), so # both cached counters are authoritative ground-truth by the time @@ -139,7 +139,7 @@ HTTP 201 # acts as the synchronisation point for the user-envelope # assertion below — the sweep is the authoritative # ground-truth for both drive- and user-side counters. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} [Options] delay: 200ms @@ -159,7 +159,7 @@ jsonpath "$[?(@.id=='{{shared_drive_id}}')].used_bytes" == 32 # If the delta path incorrectly fired the user counter, the sweep # would still correct it back to 0 (the new SQL excludes shared # drives) — this also validates the sweep formula. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} HTTP 200 @@ -207,7 +207,7 @@ jsonpath "$.storage_used_bytes" == 32 # Confirm the sweep agrees with the delta — both code paths must # give the same number. -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} HTTP 200 @@ -237,7 +237,7 @@ Authorization: Bearer {{owner_token}} HTTP 200 -POST {{base_url}}/api/admin/jobs/storage_reconcile/trigger +POST {{base_url}}/api/admin/jobs/usage_reconcile/trigger Authorization: Bearer {{admin_token}} HTTP 200