feat(job-registry): wire storage usage update

This commit is contained in:
Edouard Vanbelle
2026-07-27 22:33:04 +02:00
parent d0f4e1ef5a
commit e29f3ba067
3 changed files with 96 additions and 57 deletions
+11 -3
View File
@@ -420,8 +420,13 @@ pub trait StorageUsagePort: Send + Sync + 'static {
username: &str, username: &str,
) -> Result<i64, DomainError>; ) -> Result<i64, DomainError>;
/// Updates storage usage statistics for all users /// Reconcile every user's cached `storage_used_bytes` against the
async fn update_all_users_storage_usage(&self) -> Result<(), DomainError>; /// authoritative `SUM(drives.used_bytes)` over their personal
/// drives. Returns the number of user rows the sweep actually
/// corrected (`rows_affected` from the `UPDATE … IS DISTINCT
/// FROM …`) — used by the periodic-scheduler `JobHandler` to
/// populate `JobOutcome::Ok.count`.
async fn update_all_users_storage_usage(&self) -> Result<u64, DomainError>;
/// Checks if a user has enough quota for an additional upload. /// Checks if a user has enough quota for an additional upload.
/// Returns Ok(()) if the upload is allowed, or Err(QuotaExceeded) with a /// Returns Ok(()) if the upload is allowed, or Err(QuotaExceeded) with a
@@ -455,7 +460,10 @@ pub trait StorageUsagePort: Send + Sync + 'static {
/// `GROUP BY drive_id` aggregate, with an `IS DISTINCT FROM` /// `GROUP BY drive_id` aggregate, with an `IS DISTINCT FROM`
/// guard so idle drives don't churn dead tuples. Runs from the /// guard so idle drives don't churn dead tuples. Runs from the
/// same reconciliation ticker. /// same reconciliation ticker.
async fn update_all_drives_storage_usage(&self) -> Result<(), DomainError>; ///
/// Returns the number of drive rows actually corrected — used by
/// the periodic-scheduler `JobHandler` to populate `JobOutcome::Ok.count`.
async fn update_all_drives_storage_usage(&self) -> Result<u64, DomainError>;
/// Pre-upload quota check on a single drive. /// Pre-upload quota check on a single drive.
/// ///
@@ -3,7 +3,6 @@ use crate::common::errors::DomainError;
use crate::infrastructure::repositories::pg::UserPgRepository; use crate::infrastructure::repositories::pg::UserPgRepository;
use sqlx::PgPool; use sqlx::PgPool;
use std::sync::Arc; use std::sync::Arc;
use tokio::task;
use tracing::{debug, error, info}; use tracing::{debug, error, info};
use uuid::Uuid; use uuid::Uuid;
@@ -569,43 +568,55 @@ impl StorageUsageService {
Ok(()) Ok(())
} }
/// Spawn a background task that periodically reconciles every user's cached /// Interval helper — same clamping (min 30 s) the retired
/// `storage_used_bytes` against the actual sum of their files. /// `start_reconciliation_job` applied, exposed so DI can pass a
/// sanitised `Duration` to `JobRegistry::register`.
pub fn reconciliation_interval(interval_secs: u64) -> std::time::Duration {
std::time::Duration::from_secs(interval_secs.max(30))
}
}
pub const STORAGE_RECONCILE_JOB_NAME: &str = "storage_reconcile";
use crate::infrastructure::scheduler::{JobHandler, JobOutcome};
use async_trait::async_trait;
#[async_trait]
impl JobHandler for StorageUsageService {
fn name(&self) -> &str {
STORAGE_RECONCILE_JOB_NAME
}
/// Runs both reconciliation sweeps — drives first, then users —
/// and reports the total number of rows corrected.
/// ///
/// `GET /api/auth/me` no longer recomputes usage on the request path; this /// **Sweep order matters.** The user envelope is computed from
/// sweep (plus the per-upload update) keeps the cached value current for /// `SUM(drives.used_bytes)` over the caller's personal drives
/// all mutations — including deletes and trash — without any O(N) work on a /// (`docs/plan/drive.md` §7). Running the user sweep before the
/// hot endpoint. Runs on the maintenance pool. The first sweep is deferred /// drive sweep would read a stale per-drive counter and freeze
/// by one interval so it never adds load at boot. /// the user counter on the previous tick's numbers — invisible in
pub fn start_reconciliation_job(&self, interval_secs: u64) { /// steady state, breaks any Hurl that trashes + sweeps within one
// Floor the interval so a misconfiguration can't busy-loop the sweep. /// call. See memory note `bug_storage_sweep_order_drive_first`.
let interval_secs = interval_secs.max(30); ///
let service = self.clone(); /// Failure of one sub-sweep short-circuits the tick to `Err`;
info!( /// operators see `outcome=err, cause=handler` in the scheduler
"Starting storage-usage reconciliation job (every {}s)", /// log and the individual sweep's own `error!` line above it.
interval_secs async fn run(&self) -> JobOutcome {
); let drives = match self.update_all_drives_storage_usage().await {
task::spawn(async move { Ok(n) => n,
let mut ticker = tokio::time::interval(std::time::Duration::from_secs(interval_secs)); Err(e) => return JobOutcome::Err(format!("drive reconciliation failed: {e}")),
// tokio's first `tick()` fires immediately — consume it so the };
// first real sweep happens one interval after startup. let users = match self.update_all_users_storage_usage().await {
ticker.tick().await; Ok(n) => n,
loop { Err(e) => return JobOutcome::Err(format!("user reconciliation failed: {e}")),
ticker.tick().await; };
debug!("Running scheduled storage-usage reconciliation"); JobOutcome::ok_with(
// Drive sweep runs FIRST: the user-side sweep below drives + users,
// reads `drives.used_bytes` (the per-drive sum) to serde_json::json!({
// compute its own counter, so the drive counter must "drives_corrected": drives,
// be honest first. Failure of one is logged but "users_corrected": users,
// doesn't skip the other or the next tick. }),
if let Err(e) = service.update_all_drives_storage_usage().await { )
error!("Scheduled drive storage-usage reconciliation failed: {}", e);
}
if let Err(e) = service.update_all_users_storage_usage().await {
error!("Scheduled user storage-usage reconciliation failed: {}", e);
}
}
});
} }
} }
@@ -638,7 +649,7 @@ impl StorageUsagePort for StorageUsageService {
/// ///
/// External users are excluded — they carry no storage by construction /// External users are excluded — they carry no storage by construction
/// (DB CHECK `users_external_no_storage`). /// (DB CHECK `users_external_no_storage`).
async fn update_all_users_storage_usage(&self) -> Result<(), DomainError> { async fn update_all_users_storage_usage(&self) -> Result<u64, DomainError> {
debug!("Starting storage-usage reconciliation sweep"); debug!("Starting storage-usage reconciliation sweep");
// User envelope = SUM of `drives.used_bytes` across the user's // User envelope = SUM of `drives.used_bytes` across the user's
@@ -681,11 +692,9 @@ impl StorageUsagePort for StorageUsageService {
DomainError::internal_error("StorageUsage", format!("reconciliation sweep: {e}")) DomainError::internal_error("StorageUsage", format!("reconciliation sweep: {e}"))
})?; })?;
info!( let corrected = result.rows_affected();
"Storage-usage reconciliation corrected {} user(s)", info!("Storage-usage reconciliation corrected {} user(s)", corrected);
result.rows_affected() Ok(corrected)
);
Ok(())
} }
async fn check_storage_quota( async fn check_storage_quota(
@@ -720,7 +729,7 @@ impl StorageUsagePort for StorageUsageService {
/// FROM` guard to skip no-op rewrites so idle drives don't churn /// FROM` guard to skip no-op rewrites so idle drives don't churn
/// dead tuples. Runs from the same reconciliation ticker as the /// dead tuples. Runs from the same reconciliation ticker as the
/// user sweep; failure is logged but doesn't stop the next tick. /// user sweep; failure is logged but doesn't stop the next tick.
async fn update_all_drives_storage_usage(&self) -> Result<(), DomainError> { async fn update_all_drives_storage_usage(&self) -> Result<u64, DomainError> {
debug!("Starting drive storage-usage reconciliation sweep"); debug!("Starting drive storage-usage reconciliation sweep");
let result = sqlx::query( let result = sqlx::query(
r#" r#"
@@ -744,9 +753,10 @@ impl StorageUsagePort for StorageUsageService {
DomainError::internal_error("StorageUsage", format!("drive reconciliation sweep: {e}")) DomainError::internal_error("StorageUsage", format!("drive reconciliation sweep: {e}"))
})?; })?;
let corrected = result.rows_affected();
info!( info!(
"Drive storage-usage reconciliation corrected {} drive(s)", "Drive storage-usage reconciliation corrected {} drive(s)",
result.rows_affected() corrected
); );
// Unconditional invalidation — do NOT gate on // Unconditional invalidation — do NOT gate on
// `rows_affected() > 0`. When a fire-and-forget delta has // `rows_affected() > 0`. When a fire-and-forget delta has
@@ -762,7 +772,7 @@ impl StorageUsagePort for StorageUsageService {
// already right → zero rows → without unconditional // already right → zero rows → without unconditional
// invalidation, cache stays at the previous step's value. // invalidation, cache stays at the previous step's value.
self.invalidate_drive_lookup_caches(); self.invalidate_drive_lookup_caches();
Ok(()) Ok(corrected)
} }
async fn check_drive_quota( async fn check_drive_quota(
+29 -8
View File
@@ -1104,12 +1104,16 @@ impl AppServiceFactory {
/// ///
/// Uses the `maintenance_pool` for batch operations /// Uses the `maintenance_pool` for batch operations
/// (`update_all_users_storage_usage`) to avoid starving user requests. /// (`update_all_users_storage_usage`) to avoid starving user requests.
pub fn create_storage_usage_service( ///
/// Note: this is `async` (unlike the pre-migration version) because
/// registration with `core.job_registry` requires an `await`.
pub async fn create_storage_usage_service(
&self, &self,
_repos: &RepositoryServices, _repos: &RepositoryServices,
db_pool: &Arc<PgPool>, db_pool: &Arc<PgPool>,
maintenance_pool: &Arc<PgPool>, maintenance_pool: &Arc<PgPool>,
drive_repo: Arc<crate::infrastructure::repositories::pg::DrivePgRepository>, drive_repo: Arc<crate::infrastructure::repositories::pg::DrivePgRepository>,
core: &CoreServices,
) -> Arc<StorageUsageService> { ) -> Arc<StorageUsageService> {
let user_repository = Arc::new( let user_repository = Arc::new(
crate::infrastructure::repositories::pg::UserPgRepository::new(db_pool.clone()), crate::infrastructure::repositories::pg::UserPgRepository::new(db_pool.clone()),
@@ -1131,11 +1135,27 @@ impl AppServiceFactory {
as Arc<dyn crate::domain::repositories::drive_repository::DriveRepository>, as Arc<dyn crate::domain::repositories::drive_repository::DriveRepository>,
), ),
); );
// Keep cached storage usage fresh off the request path: GET /api/auth/me // Keep cached storage usage fresh off the request path: GET
// no longer recomputes the O(N) SUM per call; a periodic sweep does it // /api/auth/me no longer recomputes the O(N) SUM per call; a
// instead (on the maintenance pool). // periodic sweep does it instead (on the maintenance pool).
service.start_reconciliation_job(self.config.storage.usage_reconcile_secs); // Registered with the periodic-job scheduler
tracing::info!("Storage usage service initialized"); // (`docs/plan/job-registry.md` Part 1); the retired
// `start_reconciliation_job` used to spawn its own interval loop.
let interval = StorageUsageService::reconciliation_interval(
self.config.storage.usage_reconcile_secs,
);
if let Err(e) = core
.job_registry
.register(service.clone(), interval, None)
.await
{
tracing::error!("Failed to register storage_reconcile job: {e}");
} else {
tracing::info!(
"Storage-usage reconciliation registered with scheduler (interval {}s)",
interval.as_secs()
);
}
service service
} }
@@ -1315,8 +1335,9 @@ impl AppServiceFactory {
// 3c. Storage usage / quota service (needed by the instant-upload // 3c. Storage usage / quota service (needed by the instant-upload
// path inside the application services, and re-exposed on AppState // path inside the application services, and re-exposed on AppState
// for the handler-side quota checks of the byte-upload paths). // for the handler-side quota checks of the byte-upload paths).
let storage_usage = let storage_usage = self
self.create_storage_usage_service(&repos, &pool, &maintenance_pool, drive_repo.clone()); .create_storage_usage_service(&repos, &pool, &maintenance_pool, drive_repo.clone(), &core)
.await;
// 3d. Content index (embedded Tantivy) — opened before application // 3d. Content index (embedded Tantivy) — opened before application
// services so SearchService can hold the query port; the feeding // services so SearchService can hold the query port; the feeding