feat(maintenance): add a maintenance notification during backend migration

This commit is contained in:
Edouard Vanbelle
2026-08-01 18:01:10 +02:00
parent 142afecbbf
commit bbfb106a32
27 changed files with 537 additions and 23 deletions
+10
View File
@@ -2028,6 +2028,7 @@ impl AppServiceFactory {
crate::infrastructure::services::webdav_dead_property_store::create_dead_property_store(pool.clone()),
authorization: authorization.clone(),
migration_readonly: migration_readonly.clone(),
migration_progress: Arc::new(std::sync::RwLock::new(None)),
drive_repo: drive_repo.clone(),
drive_management_service: Arc::new(
crate::application::services::drive_management_service::DriveManagementService::new(
@@ -2254,6 +2255,7 @@ impl AppServiceFactory {
self.storage_path.clone(),
app_state.migration_readonly.clone(),
app_state.core.blob_backend_hot_swap.clone(),
app_state.migration_progress.clone(),
),
)
.register_recoverable_job(&app_state.core.job_registry, &job_store_provider_dyn)
@@ -2785,6 +2787,14 @@ pub struct AppState {
/// memory in sync. See `docs/plan/storage-multi-entry.md`
/// §"Read-only mode".
pub migration_readonly: Arc<std::sync::atomic::AtomicBool>,
/// Live progress snapshot for the storage-migration handler.
/// `Some(_)` while a migration is running; `None` otherwise.
/// Updated by the handler on every batch checkpoint (cheap
/// in-memory write, no DB read on the request path). The
/// server-status header middleware reads it to inform every
/// user's session banner about maintenance progress without
/// polling. See `MigrationProgress` for the field shape.
pub migration_progress: Arc<std::sync::RwLock<Option<crate::common::migration_progress::MigrationProgress>>>,
/// Drive entity repository — `GET /api/drives`, the personal-drive
/// lifecycle hook, and (post-D2) shared-drive creation flow all read
/// through this. Backing table is `storage.drives`; membership is
+66
View File
@@ -0,0 +1,66 @@
//! Shared in-memory snapshot of the running storage-migration.
//!
//! Consumed by the server-status middleware to build the
//! `X-Server-Status` response header on every authenticated
//! request. That header lets every logged-in user's session banner
//! show current maintenance progress without any polling —
//! the state travels back on the piggyback of whatever API call
//! the user was going to make anyway.
//!
//! Written by the migration handler on each batch checkpoint (a
//! cheap `RwLock::write` + a small struct copy — no DB access on
//! the request path). Cleared on `RunOutcome::Completed` /
//! `Paused` / `Failed`. `None` means "no migration is running";
//! middleware omits the header entirely in that case.
use serde::Serialize;
/// One snapshot of a running migration. Every field is a scalar so
/// the whole struct copies cheaply under the `RwLock::write` guard.
#[derive(Debug, Clone, Serialize)]
pub struct MigrationProgress {
/// The entry name blobs are being copied INTO. Used by the
/// user-facing banner text so admins/users know what the
/// server is switching to.
pub target_name: String,
/// Blobs migrated so far this run. Starts at 0 on a Fresh
/// open; on Resume the checkpointed value is loaded from the
/// run row's `stats.scanned_count`.
pub migrated_blobs: u64,
/// Total blobs in the current DB snapshot. Captured once at
/// run start via `SELECT COUNT(*) FROM storage.blobs`. Doesn't
/// change during the run — new uploads are refused while
/// read-only is engaged, so the denominator stays honest.
pub total_blobs: u64,
/// Convenience: `migrated_blobs * 100 / total_blobs`, clamped
/// to 0..=100. Middleware could compute it but it's tiny and
/// makes the JSON payload obvious.
pub percent: u8,
}
impl MigrationProgress {
pub fn new(target_name: String, total_blobs: u64) -> Self {
Self {
target_name,
migrated_blobs: 0,
total_blobs,
percent: 0,
}
}
/// Update the counter + recompute `percent`. Called by the
/// migration handler after each batch checkpoint.
pub fn bump(&mut self, migrated_delta: u64) {
self.migrated_blobs = self.migrated_blobs.saturating_add(migrated_delta);
self.recompute_percent();
}
fn recompute_percent(&mut self) {
self.percent = if self.total_blobs == 0 {
0
} else {
((self.migrated_blobs.min(self.total_blobs) as u128 * 100) / self.total_blobs as u128)
as u8
};
}
}
+1
View File
@@ -3,6 +3,7 @@ pub mod di;
pub mod errors;
pub mod fmt;
pub mod locale;
pub mod migration_progress;
pub mod mime_detect;
pub mod runtime;
pub mod stubs;
@@ -121,6 +121,12 @@ pub struct StorageMigrationService {
/// delegates through.
blob_backend_hot_swap:
Arc<crate::infrastructure::services::swappable_blob_backend::SwappableBlobBackend>,
/// Shared in-memory progress snapshot. `Some(_)` during a
/// running/paused migration, `None` otherwise. Read by the
/// server-status header middleware to broadcast maintenance
/// state to every user's session without polling.
migration_progress:
Arc<std::sync::RwLock<Option<crate::common::migration_progress::MigrationProgress>>>,
}
impl StorageMigrationService {
@@ -135,6 +141,9 @@ impl StorageMigrationService {
blob_backend_hot_swap: Arc<
crate::infrastructure::services::swappable_blob_backend::SwappableBlobBackend,
>,
migration_progress: Arc<
std::sync::RwLock<Option<crate::common::migration_progress::MigrationProgress>>,
>,
) -> Self {
Self {
pool,
@@ -144,6 +153,7 @@ impl StorageMigrationService {
storage_path_fallback,
migration_readonly,
blob_backend_hot_swap,
migration_progress,
}
}
@@ -384,6 +394,27 @@ impl RecoverableJobHandler for StorageMigrationService {
cutover hot-swap completes"
);
// Seed the shared progress snapshot for the header
// middleware. Total blob count is a one-shot SELECT COUNT(*)
// — best-effort; if it fails we still push a snapshot with
// total=0 so the banner at least shows *something* is
// happening.
let total_blobs: u64 = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM storage.blobs")
.fetch_one(self.pool.as_ref())
.await
.map(|n| n.max(0) as u64)
.unwrap_or(0);
{
let mut guard = self
.migration_progress
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*guard = Some(crate::common::migration_progress::MigrationProgress::new(
target_name.clone(),
total_blobs,
));
}
let source_kind = self.source.backend_type();
let target_kind = target.backend_type();
tracing::info!(
@@ -613,6 +644,19 @@ impl RecoverableJobHandler for StorageMigrationService {
message: format!("checkpoint: {e}"),
};
}
// Bump the shared progress snapshot so the server-status
// header middleware surfaces fresh numbers on every
// user's next API call. Guard is held only for a struct
// update — microseconds.
{
let mut guard = self
.migration_progress
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(progress) = guard.as_mut() {
progress.bump(batch_len);
}
}
if (rows.len() as i64) < BATCH_SIZE {
return self
@@ -703,6 +747,16 @@ impl StorageMigrationService {
.await
.is_ok();
self.migration_readonly.store(false, Ordering::Relaxed);
// Clear the shared progress snapshot so the server-status
// header stops emitting on subsequent requests. Guard held
// only for the assignment.
{
let mut guard = self
.migration_progress
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*guard = None;
}
if !readonly_persisted {
tracing::warn!(
+18 -7
View File
@@ -687,13 +687,24 @@ pub fn create_api_routes(app_state: &Arc<AppState>) -> Router<Arc<AppState>> {
// them on every overlapping request.
router = router.route("/{*rest}", any(api_not_found));
// No per-router layers: the global `TraceLayer` + request-id stack in
// `main.rs` wraps the whole app (this `/api` router is nested into it),
// so a second `TraceLayer` here just double-wrapped every `/api`
// request in a redundant span + response-future poll (benches/ROUND13.md
// §H1). Compression is likewise the global layer's job — re-applying it
// here (no predicate) would compress media downloads, burning CPU for
// ~0 gain and stripping `Content-Length`.
// Server-status header. Stamps `X-Server-Status` on every
// response so the frontend's fetch wrapper can update a
// reactive store — banner shows/hides without polling.
// Sub-nanosecond on the hot path (single atomic load), a few
// µs on the cold path (only during a running migration). See
// `middleware::server_status`.
let router = router.layer(axum::middleware::from_fn_with_state(
app_state.clone(),
crate::interfaces::middleware::server_status::server_status_middleware,
));
// No per-router layers beyond that: the global `TraceLayer` + request-id
// stack in `main.rs` wraps the whole app (this `/api` router is nested
// into it), so a second `TraceLayer` here just double-wrapped every
// `/api` request in a redundant span + response-future poll
// (benches/ROUND13.md §H1). Compression is likewise the global layer's
// job — re-applying it here (no predicate) would compress media
// downloads, burning CPU for ~0 gain and stripping `Content-Length`.
router
}
+1
View File
@@ -3,6 +3,7 @@ pub mod auth;
pub mod csrf;
pub mod locale;
pub mod rate_limit;
pub mod server_status;
pub mod trace_span;
pub mod trusted_proxy;
pub mod user;
+119
View File
@@ -0,0 +1,119 @@
//! Middleware that stamps `X-Server-Status` on every response.
//!
//! Consumed by the frontend `apiFetch` wrapper — every API round-trip
//! carries the current server maintenance state back to the client
//! (no polling, no dedicated endpoint). The banner in the app shell
//! subscribes to a store the wrapper updates and shows/hides itself
//! reactively. See `docs/plan/storage-multi-entry.md` §"Read-only mode"
//! for the broader design.
//!
//! ## Cost model
//!
//! On the *hot path* (no migration running — the ~100% case in normal
//! operation) this middleware does:
//! 1. one `AtomicBool::load(Relaxed)` — sub-nanosecond;
//! 2. an early return when `false`.
//!
//! No allocation, no lock, no formatting. Adds no measurable latency
//! at any user count.
//!
//! On the *cold path* (migration in progress) this middleware does:
//! 1. the atomic load above;
//! 2. one `RwLock::read` (uncontended — writers are the migration
//! handler, one per batch every ~100 blobs);
//! 3. one small `serde_json::to_string` call on a 4-field struct
//! (a few dozen bytes);
//! 4. one header insertion.
//!
//! Total per-request work in this branch: microseconds.
use axum::extract::Request;
use axum::extract::State;
use axum::http::HeaderValue;
use axum::middleware::Next;
use axum::response::Response;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use crate::common::di::AppState;
/// Name of the response header the frontend reads. Kept short — an
/// admin browser session may keep this header around in every open
/// tab's dev-tools network view during a migration; the value is
/// small JSON but the name should not add bloat.
pub const SERVER_STATUS_HEADER: &str = "x-server-status";
/// Compact JSON shape written into the header. Fields are documented
/// in `common::migration_progress::MigrationProgress`.
///
/// Kept internal so the wire format can evolve. Frontend treats the
/// header as opaque JSON and pattern-matches on the fields it
/// currently understands.
#[derive(serde::Serialize)]
struct HeaderPayload {
readonly: bool,
#[serde(skip_serializing_if = "Option::is_none")]
migration: Option<MigrationHeader>,
}
#[derive(serde::Serialize)]
struct MigrationHeader {
// `target` is owned here — the RwLock guard is released before
// serialisation, so a borrowed slice wouldn't survive. Names
// are small (`[a-z0-9_-]{1,32}`) so the copy is trivial.
target: String,
migrated: u64,
total: u64,
percent: u8,
}
pub async fn server_status_middleware(
State(state): State<Arc<AppState>>,
request: Request,
next: Next,
) -> Response {
// Hot-path fast return. When no migration is running the flag is
// false and there's nothing to emit — a bare atomic load and out.
let readonly = state.migration_readonly.load(Ordering::Relaxed);
let mut response = next.run(request).await;
if !readonly {
return response;
}
// Cold path — build the payload from the shared progress
// snapshot. If the snapshot is absent (readonly is true but the
// handler hasn't seeded progress yet, or a restart-during-
// migration scenario) we still emit `readonly: true` so the
// banner shows — the frontend renders a "maintenance in progress"
// message even when specific numbers aren't available.
let payload = {
let guard = state
.migration_progress
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
HeaderPayload {
readonly: true,
migration: guard.as_ref().map(|p| MigrationHeader {
target: p.target_name.clone(),
migrated: p.migrated_blobs,
total: p.total_blobs,
percent: p.percent,
}),
}
};
// `serde_json::to_string` on this 4-field struct is a few
// dozen-byte allocation — negligible against the response body.
// A serialize failure here would be a programming bug (all
// fields are trivially serializable), so we degrade to a
// minimal `readonly: true` string rather than skipping the
// header entirely.
let value =
serde_json::to_string(&payload).unwrap_or_else(|_| r#"{"readonly":true}"#.to_string());
if let Ok(header_value) = HeaderValue::from_str(&value) {
response
.headers_mut()
.insert(SERVER_STATUS_HEADER, header_value);
}
response
}