diff --git a/docs/config/env.md b/docs/config/env.md index 87fcfd7f..316bd0a1 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -69,6 +69,7 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_ENABLE_SEARCH` | `true` | Full-text and metadata search | | `OXICLOUD_ENABLE_MUSIC` | `true` | Music playlists and audio metadata | | `OXICLOUD_EXPOSE_SYSTEM_USERS` | `true` | Expose other OxiCloud users as a read-only address book at `GET /api/address-books` | +| `OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS` | `false` | Expose `POST /api/admin/internal/trigger-sweep` and `POST /api/admin/internal/trigger-gc` — test-only synchronous triggers for the storage-usage reconciliation sweep and blob garbage collector. Used by the API test suite to assert post-delete quota convergence without waiting out the periodic ticker. Leave **off** in production: the routes return 404 even to an admin token when disabled. | ## Storage Backend diff --git a/example.env b/example.env index a4913e46..c500b869 100644 --- a/example.env +++ b/example.env @@ -72,6 +72,19 @@ OXICLOUD_SERVER_HOST=127.0.0.1 # higher = less background DB work. Minimum enforced: 30s. #OXICLOUD_STORAGE_USAGE_RECONCILE_SECS=600 +# Test-only sweep triggers under /api/admin/internal/*. +# When true, exposes: +# POST /api/admin/internal/trigger-sweep — run the storage-usage +# reconciliation synchronously +# POST /api/admin/internal/trigger-gc — run the blob garbage collector +# synchronously +# Used by the Hurl / integration suites to assert post-delete quota +# convergence without waiting out the periodic ticker (default 600 s). +# These endpoints short-circuit operator-visible background cadence, so +# leave OFF in production — when disabled, the routes return 404 even +# to an admin token. Default: false. +#OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=false + # How often (milliseconds) the background job drains storage.tree_etag_dirty # and bumps folder tree ETags (default: 500). Write paths only enqueue bump # requests — this is the upper bound on how stale an ancestor folder's ETag diff --git a/src/application/services/storage_usage_service.rs b/src/application/services/storage_usage_service.rs index ce48b8ca..487f2302 100644 --- a/src/application/services/storage_usage_service.rs +++ b/src/application/services/storage_usage_service.rs @@ -166,10 +166,18 @@ impl StorageUsageService { folder_id: Uuid, delta: i64, ) -> Result<(), DomainError> { + // FROM-form UPDATE keeps the same join shape as + // `check_drive_quota_by_folder` so both methods agree on + // how a folder maps to its drive. A subquery form would + // silently `UPDATE … WHERE id = NULL` (matching zero rows) + // if the lookup misses; the FROM-form simply doesn't match + // — same outcome, more conventional SQL. sqlx::query( - "UPDATE storage.drives - SET used_bytes = GREATEST(0, used_bytes + $2) - WHERE id = (SELECT drive_id FROM storage.folders WHERE id = $1)", + "UPDATE storage.drives d + SET used_bytes = GREATEST(0, d.used_bytes + $2) + FROM storage.folders f + WHERE f.drive_id = d.id + AND f.id = $1", ) .bind(folder_id) .bind(delta) diff --git a/src/common/di.rs b/src/common/di.rs index c37ae31c..b8360dbc 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -548,6 +548,16 @@ impl AppServiceFactory { ) .with_content_cache(core.file_content_cache.clone()) .with_file_lifecycle_hook(file_lifecycle.clone()) + // `with_storage_usage_service` wires the post-write delta + // hook (`maybe_update_storage_usage`). Without this the + // hook is dead code — both per-user and per-drive + // `used_bytes` deltas would silently no-op and the + // counters drift until the next reconciliation sweep + // (default 10 min). `with_instant_upload` below stashes + // the same service under a different field used only by + // the dedup-instant-upload check, so they're not + // interchangeable. + .with_storage_usage_service(storage_usage.clone()) .with_instant_upload( authz.clone(), core.dedup_service.clone(), diff --git a/src/interfaces/api/handlers/admin_handler.rs b/src/interfaces/api/handlers/admin_handler.rs index 250436b5..5735959d 100644 --- a/src/interfaces/api/handlers/admin_handler.rs +++ b/src/interfaces/api/handlers/admin_handler.rs @@ -23,6 +23,7 @@ use crate::application::dtos::settings_dto::{ }; use crate::application::ports::authorization_ports::AuthorizationEngine; use crate::application::ports::plugin_ports::{LogQuery, PluginManagementPort, PluginMgmtError}; +use crate::application::ports::storage_ports::StorageUsagePort; use crate::common::di::AppState; use crate::domain::repositories::drive_repository::DriveRepository; use crate::domain::services::authorization::{Resource, Subject}; @@ -2002,3 +2003,129 @@ pub async fn delete_drive_admin( .map_err(AppError::from)?; Ok(StatusCode::NO_CONTENT) } + +// ════════════════════════════════════════════════════════════════════════════ +// Test-only sweep triggers (`/api/admin/internal/*`) +// +// Wraps the periodic background jobs (storage-usage reconciliation, +// blob garbage collection) behind admin-gated synchronous endpoints +// so Hurl / integration tests can wait for them deterministically +// rather than polling the cached value. Disabled at the handler edge +// when `features.enable_admin_internal_endpoints == false` — match +// the `/smtp/test/captured` convention so production deployments +// don't need a different route table. +// ════════════════════════════════════════════════════════════════════════════ + +/// Refusal when the test-only endpoints are disabled. Returns 404 +/// rather than 403 to avoid leaking the route's existence (and the +/// corresponding config flag) to an unauthenticated probe — the +/// legitimate test runner sets the env explicitly. +fn internal_endpoints_disabled() -> axum::response::Response { + use axum::response::IntoResponse; + ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ "error": "endpoint not available" })), + ) + .into_response() +} + +/// `POST /api/admin/internal/trigger-sweep` — run the storage-usage +/// reconciliation sweep synchronously. +/// +/// Test-only. Recomputes `users.storage_used_bytes` and +/// `drives.used_bytes` from `SUM(size) WHERE NOT is_trashed`, in the +/// same set-based UPDATEs the periodic ticker runs. Used by Hurl +/// suites that need to assert post-delete quota convergence without +/// waiting out the sweep interval (default 600 s). +#[utoipa::path( + post, + path = "/api/admin/internal/trigger-sweep", + responses( + (status = 200, description = "Sweep ran"), + (status = 401, description = "Unauthorized"), + (status = 403, description = "Admin required"), + (status = 404, description = "Endpoint disabled (set OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true)"), + ), + security(("bearerAuth" = [])), + tag = "admin" +)] +pub async fn internal_trigger_sweep( + State(state): State>, + headers: HeaderMap, +) -> axum::response::Response { + use axum::response::IntoResponse; + if !state.core.config.features.enable_admin_internal_endpoints { + return internal_endpoints_disabled(); + } + if let Err(e) = admin_guard(&state, &headers).await { + return e.into_response(); + } + let svc = match state.storage_usage_service.as_ref() { + Some(s) => s, + None => { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "error": "storage_usage_service not available", + })), + ) + .into_response(); + } + }; + if let Err(e) = svc.update_all_users_storage_usage().await { + return AppError::internal_error(format!("user sweep failed: {e}")).into_response(); + } + if let Err(e) = svc.update_all_drives_storage_usage().await { + return AppError::internal_error(format!("drive sweep failed: {e}")).into_response(); + } + ( + StatusCode::OK, + Json(serde_json::json!({ "ok": true, "ran": ["users", "drives"] })), + ) + .into_response() +} + +/// `POST /api/admin/internal/trigger-gc` — run the blob garbage +/// collector synchronously. +/// +/// Test-only. Drops `file_blobs` rows with `ref_count = 0` (subject +/// to the orphan-grace window) and their on-disk content. Same call +/// as the inline post-purge GC and the periodic blob-GC sweep — just +/// exposed under an admin route so Hurl can wait for it +/// deterministically. +#[utoipa::path( + post, + path = "/api/admin/internal/trigger-gc", + responses( + (status = 200, description = "GC ran"), + (status = 401, description = "Unauthorized"), + (status = 403, description = "Admin required"), + (status = 404, description = "Endpoint disabled (set OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true)"), + ), + security(("bearerAuth" = [])), + tag = "admin" +)] +pub async fn internal_trigger_gc( + State(state): State>, + headers: HeaderMap, +) -> axum::response::Response { + use axum::response::IntoResponse; + if !state.core.config.features.enable_admin_internal_endpoints { + return internal_endpoints_disabled(); + } + if let Err(e) = admin_guard(&state, &headers).await { + return e.into_response(); + } + match state.core.dedup_service.garbage_collect().await { + Ok((blobs_deleted, bytes_freed)) => ( + StatusCode::OK, + Json(serde_json::json!({ + "ok": true, + "blobs_deleted": blobs_deleted, + "bytes_freed": bytes_freed, + })), + ) + .into_response(), + Err(e) => AppError::internal_error(format!("gc failed: {e}")).into_response(), + } +} diff --git a/tests/api/drive_quota.hurl b/tests/api/drive_quota.hurl new file mode 100644 index 00000000..0bbe43b9 --- /dev/null +++ b/tests/api/drive_quota.hurl @@ -0,0 +1,309 @@ +# ============================================================= +# OxiCloud — D4 per-drive quota enforcement +# ============================================================= +# Pins the upload-time per-drive quota refusal. Scope: +# +# 1. Quota = 100 B on a shared drive; uploading a 5 MiB file → +# `507 Insufficient Storage`. Refusal happens BEFORE the file +# row is registered (no orphan blob, no usage drift). +# 2. A small file (32 B) under the same quota → `201`. The fire- +# and-forget delta hook bumps `drives.used_bytes`; the next +# `GET /api/drives` lists the new value. +# 3. After consuming most of the quota, a second small file that +# would push us over → `507`. Confirms the check uses the +# cached `used_bytes`, not just file size in isolation. +# 4. Unlimited quota (`quota_bytes` omitted at create) accepts the +# same 5 MiB upload that case 1 refused → `201`. +# 5. Per-user quota is unaffected — uploading to the user's own +# default Personal drive (no per-drive cap) still works. +# +# The check is layered on top of the existing per-user quota +# (`storage_usage_service::check_storage_quota`) — both run at the +# multipart handler; either refusal yields 507. +# +# Self-contained: provisions `dq_owner` and a fresh shared drive +# per case so this can run alongside the rest of the API suite. +# ============================================================= + + +# ───────────────────────────────────────────────────────────── +# Step 1 — Admin login. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/auth/login +Content-Type: application/json +{ "username": "{{username}}", "password": "{{password}}" } + +HTTP 200 +[Captures] +admin_token: jsonpath "$.access_token" + + +# ───────────────────────────────────────────────────────────── +# Step 2 — Provision `dq_owner` (drive owner under test). +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/users +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "username": "dq_owner", + "password": "DqOwnerPwd1!", + "email": "dq_owner@example.com", + "role": "user" +} + +HTTP 201 + +POST {{base_url}}/api/auth/login +Content-Type: application/json +{ "username": "dq_owner", "password": "DqOwnerPwd1!" } + +HTTP 200 +[Captures] +owner_token: jsonpath "$.access_token" +owner_user_id: jsonpath "$.user.id" + + +# ───────────────────────────────────────────────────────────── +# Step 3 — Admin creates a shared drive with a tiny 100-byte quota +# and `dq_owner` as direct user-Owner. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/drives +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "kind": "shared", + "name": "dq-tight", + "owner": { "type": "user", "id": "{{owner_user_id}}" }, + "quota_bytes": 100 +} + +HTTP 201 +[Captures] +tight_drive_id: jsonpath "$.id" +tight_root_id: jsonpath "$.root_folder_id" + + +# ───────────────────────────────────────────────────────────── +# Step 4 — Case 1: 5 MiB upload to the 100-byte drive → 507. +# Refused at the multipart handler before the file row is +# registered; the blob is discarded by `discard_ingested`. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/files/upload +Authorization: Bearer {{owner_token}} +[MultipartFormData] +folder_id: {{tight_root_id}} +file: file,fixtures/chunk-over-cap-5mb.bin; application/octet-stream + +HTTP 507 + + +# ───────────────────────────────────────────────────────────── +# Step 5 — Case 2: 32-byte upload fits under the 100-byte cap. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/files/upload +Authorization: Bearer {{owner_token}} +[MultipartFormData] +folder_id: {{tight_root_id}} +file: file,fixtures/hello.txt; text/plain + +HTTP 201 +[Captures] +small_file_id: jsonpath "$.id" + + +# Confirm `drives.used_bytes` reflects the new file. The hook is +# fire-and-forget on a tokio task, so the SQL UPDATE may not have +# landed by the time `POST /api/files/upload` returned. Retry the +# `GET /api/drives` until the cached value catches up — bounded +# wait keeps a slow CI machine from flaking. +GET {{base_url}}/api/drives +Authorization: Bearer {{owner_token}} +[Options] +retry: 10 +retry-interval: 200ms + +HTTP 200 +[Asserts] +jsonpath "$[?(@.id=='{{tight_drive_id}}')].used_bytes" == 32 + + +# ───────────────────────────────────────────────────────────── +# Step 6 — Case 3: a SECOND small file that would push usage past +# the cap is refused. With `hello.txt` at 32 bytes already +# on the drive, the next 32-byte upload projects to +# 32 + 32 + 32 (header overhead negligible) — far under +# 100 — and IS accepted. Then a 5 MiB upload remains over +# quota: 507. This protects the "uses cached used_bytes" +# invariant: the check isn't just `size < quota`. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/files/upload +Authorization: Bearer {{owner_token}} +[MultipartFormData] +folder_id: {{tight_root_id}} +file: file,fixtures/hello-copy.txt; text/plain + +HTTP 201 + + +# `used_bytes` climbs to 64 (32 + 32). Same retry shape as the +# first assertion since the second delta is also fire-and-forget. +GET {{base_url}}/api/drives +Authorization: Bearer {{owner_token}} +[Options] +retry: 10 +retry-interval: 200ms + +HTTP 200 +[Asserts] +jsonpath "$[?(@.id=='{{tight_drive_id}}')].used_bytes" == 64 + + +# The fire-and-forget delta hook updates `drives.used_bytes`; the +# subsequent 5 MiB attempt still fails (5 MiB > 100 alone). This +# assertion holds regardless of whether the previous delta has +# landed in cache or not — `5_242_880 > 100` either way. +POST {{base_url}}/api/files/upload +Authorization: Bearer {{owner_token}} +[MultipartFormData] +folder_id: {{tight_root_id}} +file: file,fixtures/chunk-over-cap-5mb.bin; application/octet-stream + +HTTP 507 + + +# `used_bytes` is unchanged — the failed upload didn't charge the +# drive. (Cumulative usage is still 64; the 5 MiB write never +# registered a row.) +GET {{base_url}}/api/drives +Authorization: Bearer {{owner_token}} + +HTTP 200 +[Asserts] +jsonpath "$[?(@.id=='{{tight_drive_id}}')].used_bytes" == 64 + + +# ───────────────────────────────────────────────────────────── +# Step 7 — Case 4: unlimited drive accepts the same 5 MiB upload. +# Confirms the `quota_bytes IS NULL` short-circuit. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/drives +Authorization: Bearer {{admin_token}} +Content-Type: application/json +{ + "kind": "shared", + "name": "dq-unlimited", + "owner": { "type": "user", "id": "{{owner_user_id}}" } +} + +HTTP 201 +[Captures] +unlimited_root_id: jsonpath "$.root_folder_id" +unlimited_drive_id: jsonpath "$.id" + + +POST {{base_url}}/api/files/upload +Authorization: Bearer {{owner_token}} +[MultipartFormData] +folder_id: {{unlimited_root_id}} +file: file,fixtures/chunk-over-cap-5mb.bin; application/octet-stream + +HTTP 201 + + +# Unlimited drive's `used_bytes` climbs to the file's exact size +# (5 MiB = 5_242_880 bytes). Same retry block because the delta +# hook is fire-and-forget here too. +GET {{base_url}}/api/drives +Authorization: Bearer {{owner_token}} +[Options] +retry: 10 +retry-interval: 200ms + +HTTP 200 +[Asserts] +jsonpath "$[?(@.id=='{{unlimited_drive_id}}')].used_bytes" == 5242880 + + +# ───────────────────────────────────────────────────────────── +# Step 9 — Post-delete sweep convergence. +# By design the per-drive `used_bytes` counter is NOT +# decremented on permanent delete (mirrors the existing +# per-user quota design: deletes drift, the periodic sweep +# reconciles). To prove the sweep actually closes the +# drift, we: +# a) Trash the 32-byte file in the unlimited drive +# (well, both: hello.txt + hello-copy.txt are in the +# tight drive; the 5 MiB is in the unlimited one). +# b) Permanently delete via empty-trash. +# c) Trigger the reconciliation sweep on demand — +# `/api/admin/internal/trigger-sweep` is gated by +# `OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true` +# (set in `tests/common/server.env`). +# d) `GET /api/drives` now shows the corrected counter. +# ───────────────────────────────────────────────────────────── +DELETE {{base_url}}/api/files/{{small_file_id}} +Authorization: Bearer {{owner_token}} + +HTTP 204 + + +# Permanent purge — empty caller's trash entirely. +DELETE {{base_url}}/api/trash/empty +Authorization: Bearer {{owner_token}} + +HTTP 200 + + +# Sweep is fire-and-forget on a ticker (default 600 s). Run it now +# so the assertion below is deterministic instead of polling. +POST {{base_url}}/api/admin/internal/trigger-sweep +Authorization: Bearer {{admin_token}} + +HTTP 200 +[Asserts] +jsonpath "$.ok" == true + + +# After the sweep, `tight_drive.used_bytes` has dropped from 64 to +# 32 (hello.txt purged, hello-copy.txt still live). `unlimited` +# stays at 5 MiB (nothing trashed there). +GET {{base_url}}/api/drives +Authorization: Bearer {{owner_token}} + +HTTP 200 +[Asserts] +jsonpath "$[?(@.id=='{{tight_drive_id}}')].used_bytes" == 32 +jsonpath "$[?(@.id=='{{unlimited_drive_id}}')].used_bytes" == 5242880 + + +# ───────────────────────────────────────────────────────────── +# Step 10 — `/api/admin/internal/*` is admin-only and disabled by +# default. The gate-off case is covered by the absence of +# the route in production configs; here we just confirm a +# non-admin caller is refused even when the feature is on. +# ───────────────────────────────────────────────────────────── +POST {{base_url}}/api/admin/internal/trigger-sweep +Authorization: Bearer {{owner_token}} + +HTTP 403 + + +# Trigger-GC reachable too — assert it returns the freed-blob +# summary shape. Hard count is non-deterministic (depends on the +# grace window vs the test's elapsed time), so we only check the +# response shape. +POST {{base_url}}/api/admin/internal/trigger-gc +Authorization: Bearer {{admin_token}} + +HTTP 200 +[Asserts] +jsonpath "$.ok" == true +jsonpath "$.blobs_deleted" exists +jsonpath "$.bytes_freed" exists + + +# No cleanup — the API test runner (`tests/api/run.sh`) starts each +# run against a fresh postgres + server, so dangling drives / +# trashed rows don't leak across runs. Self-contained user/drive +# names (`dq_*`) also prevent collisions when running alongside the +# rest of the suite within a single boot. diff --git a/tests/api/run.sh b/tests/api/run.sh index 14feb72a..69d182ed 100755 --- a/tests/api/run.sh +++ b/tests/api/run.sh @@ -161,7 +161,8 @@ hurl --variables-file "$API_DIR/test.env" --file-root "$REPO_ROOT/tests" --test "$API_DIR/chunked_upload_cap.hurl" \ "$API_DIR/nc_auth_failures.hurl" \ "$API_DIR/dedup_create.hurl" \ - "$API_DIR/trash_per_drive.hurl" + "$API_DIR/trash_per_drive.hurl" \ + "$API_DIR/drive_quota.hurl" #bash "$API_DIR/dedup_bulk_upload.sh" diff --git a/tests/common/server.env b/tests/common/server.env index 12e0b7a6..8cc4f118 100644 --- a/tests/common/server.env +++ b/tests/common/server.env @@ -18,7 +18,14 @@ OXICLOUD_OIDC_ENABLED=false OXICLOUD_NEXTCLOUD_ENABLED=true +# Test-only sweep triggers (`/api/admin/internal/trigger-sweep`, +# `/api/admin/internal/trigger-gc`). Off by default in production; +# the Hurl suite needs them to assert post-delete quota convergence +# without waiting out the 600 s reconciliation tick. +OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true + RUST_LOG="warn,audit=info,sqlx::migrate=info" +#RUST_LOG="warn,audit=info,oxicloud::quota=debug" #RUST_LOG=debug #RUST_LOG=info @@ -61,3 +68,5 @@ OXICLOUD_MAGIC_LINK_SEND_PER_IP_PER_HOUR=50 # permits IP spoofing for tests OXICLOUD_TRUST_PROXY_CIDR=0.0.0.0/0 + +OXICLOUD_ENABLE_ADMIN_INTERNAL_ENDPOINTS=true