feat(maintenance): add a maintenance notification during backend migration
This commit is contained in:
@@ -2028,6 +2028,7 @@ impl AppServiceFactory {
|
||||
crate::infrastructure::services::webdav_dead_property_store::create_dead_property_store(pool.clone()),
|
||||
authorization: authorization.clone(),
|
||||
migration_readonly: migration_readonly.clone(),
|
||||
migration_progress: Arc::new(std::sync::RwLock::new(None)),
|
||||
drive_repo: drive_repo.clone(),
|
||||
drive_management_service: Arc::new(
|
||||
crate::application::services::drive_management_service::DriveManagementService::new(
|
||||
@@ -2254,6 +2255,7 @@ impl AppServiceFactory {
|
||||
self.storage_path.clone(),
|
||||
app_state.migration_readonly.clone(),
|
||||
app_state.core.blob_backend_hot_swap.clone(),
|
||||
app_state.migration_progress.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&app_state.core.job_registry, &job_store_provider_dyn)
|
||||
@@ -2785,6 +2787,14 @@ pub struct AppState {
|
||||
/// memory in sync. See `docs/plan/storage-multi-entry.md`
|
||||
/// §"Read-only mode".
|
||||
pub migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||
/// Live progress snapshot for the storage-migration handler.
|
||||
/// `Some(_)` while a migration is running; `None` otherwise.
|
||||
/// Updated by the handler on every batch checkpoint (cheap
|
||||
/// in-memory write, no DB read on the request path). The
|
||||
/// server-status header middleware reads it to inform every
|
||||
/// user's session banner about maintenance progress without
|
||||
/// polling. See `MigrationProgress` for the field shape.
|
||||
pub migration_progress: Arc<std::sync::RwLock<Option<crate::common::migration_progress::MigrationProgress>>>,
|
||||
/// Drive entity repository — `GET /api/drives`, the personal-drive
|
||||
/// lifecycle hook, and (post-D2) shared-drive creation flow all read
|
||||
/// through this. Backing table is `storage.drives`; membership is
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Shared in-memory snapshot of the running storage-migration.
|
||||
//!
|
||||
//! Consumed by the server-status middleware to build the
|
||||
//! `X-Server-Status` response header on every authenticated
|
||||
//! request. That header lets every logged-in user's session banner
|
||||
//! show current maintenance progress without any polling —
|
||||
//! the state travels back on the piggyback of whatever API call
|
||||
//! the user was going to make anyway.
|
||||
//!
|
||||
//! Written by the migration handler on each batch checkpoint (a
|
||||
//! cheap `RwLock::write` + a small struct copy — no DB access on
|
||||
//! the request path). Cleared on `RunOutcome::Completed` /
|
||||
//! `Paused` / `Failed`. `None` means "no migration is running";
|
||||
//! middleware omits the header entirely in that case.
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
/// One snapshot of a running migration. Every field is a scalar so
|
||||
/// the whole struct copies cheaply under the `RwLock::write` guard.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct MigrationProgress {
|
||||
/// The entry name blobs are being copied INTO. Used by the
|
||||
/// user-facing banner text so admins/users know what the
|
||||
/// server is switching to.
|
||||
pub target_name: String,
|
||||
/// Blobs migrated so far this run. Starts at 0 on a Fresh
|
||||
/// open; on Resume the checkpointed value is loaded from the
|
||||
/// run row's `stats.scanned_count`.
|
||||
pub migrated_blobs: u64,
|
||||
/// Total blobs in the current DB snapshot. Captured once at
|
||||
/// run start via `SELECT COUNT(*) FROM storage.blobs`. Doesn't
|
||||
/// change during the run — new uploads are refused while
|
||||
/// read-only is engaged, so the denominator stays honest.
|
||||
pub total_blobs: u64,
|
||||
/// Convenience: `migrated_blobs * 100 / total_blobs`, clamped
|
||||
/// to 0..=100. Middleware could compute it but it's tiny and
|
||||
/// makes the JSON payload obvious.
|
||||
pub percent: u8,
|
||||
}
|
||||
|
||||
impl MigrationProgress {
|
||||
pub fn new(target_name: String, total_blobs: u64) -> Self {
|
||||
Self {
|
||||
target_name,
|
||||
migrated_blobs: 0,
|
||||
total_blobs,
|
||||
percent: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Update the counter + recompute `percent`. Called by the
|
||||
/// migration handler after each batch checkpoint.
|
||||
pub fn bump(&mut self, migrated_delta: u64) {
|
||||
self.migrated_blobs = self.migrated_blobs.saturating_add(migrated_delta);
|
||||
self.recompute_percent();
|
||||
}
|
||||
|
||||
fn recompute_percent(&mut self) {
|
||||
self.percent = if self.total_blobs == 0 {
|
||||
0
|
||||
} else {
|
||||
((self.migrated_blobs.min(self.total_blobs) as u128 * 100) / self.total_blobs as u128)
|
||||
as u8
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ pub mod di;
|
||||
pub mod errors;
|
||||
pub mod fmt;
|
||||
pub mod locale;
|
||||
pub mod migration_progress;
|
||||
pub mod mime_detect;
|
||||
pub mod runtime;
|
||||
pub mod stubs;
|
||||
|
||||
@@ -121,6 +121,12 @@ pub struct StorageMigrationService {
|
||||
/// delegates through.
|
||||
blob_backend_hot_swap:
|
||||
Arc<crate::infrastructure::services::swappable_blob_backend::SwappableBlobBackend>,
|
||||
/// Shared in-memory progress snapshot. `Some(_)` during a
|
||||
/// running/paused migration, `None` otherwise. Read by the
|
||||
/// server-status header middleware to broadcast maintenance
|
||||
/// state to every user's session without polling.
|
||||
migration_progress:
|
||||
Arc<std::sync::RwLock<Option<crate::common::migration_progress::MigrationProgress>>>,
|
||||
}
|
||||
|
||||
impl StorageMigrationService {
|
||||
@@ -135,6 +141,9 @@ impl StorageMigrationService {
|
||||
blob_backend_hot_swap: Arc<
|
||||
crate::infrastructure::services::swappable_blob_backend::SwappableBlobBackend,
|
||||
>,
|
||||
migration_progress: Arc<
|
||||
std::sync::RwLock<Option<crate::common::migration_progress::MigrationProgress>>,
|
||||
>,
|
||||
) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
@@ -144,6 +153,7 @@ impl StorageMigrationService {
|
||||
storage_path_fallback,
|
||||
migration_readonly,
|
||||
blob_backend_hot_swap,
|
||||
migration_progress,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -384,6 +394,27 @@ impl RecoverableJobHandler for StorageMigrationService {
|
||||
cutover hot-swap completes"
|
||||
);
|
||||
|
||||
// Seed the shared progress snapshot for the header
|
||||
// middleware. Total blob count is a one-shot SELECT COUNT(*)
|
||||
// — best-effort; if it fails we still push a snapshot with
|
||||
// total=0 so the banner at least shows *something* is
|
||||
// happening.
|
||||
let total_blobs: u64 = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM storage.blobs")
|
||||
.fetch_one(self.pool.as_ref())
|
||||
.await
|
||||
.map(|n| n.max(0) as u64)
|
||||
.unwrap_or(0);
|
||||
{
|
||||
let mut guard = self
|
||||
.migration_progress
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
*guard = Some(crate::common::migration_progress::MigrationProgress::new(
|
||||
target_name.clone(),
|
||||
total_blobs,
|
||||
));
|
||||
}
|
||||
|
||||
let source_kind = self.source.backend_type();
|
||||
let target_kind = target.backend_type();
|
||||
tracing::info!(
|
||||
@@ -613,6 +644,19 @@ impl RecoverableJobHandler for StorageMigrationService {
|
||||
message: format!("checkpoint: {e}"),
|
||||
};
|
||||
}
|
||||
// Bump the shared progress snapshot so the server-status
|
||||
// header middleware surfaces fresh numbers on every
|
||||
// user's next API call. Guard is held only for a struct
|
||||
// update — microseconds.
|
||||
{
|
||||
let mut guard = self
|
||||
.migration_progress
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(progress) = guard.as_mut() {
|
||||
progress.bump(batch_len);
|
||||
}
|
||||
}
|
||||
|
||||
if (rows.len() as i64) < BATCH_SIZE {
|
||||
return self
|
||||
@@ -703,6 +747,16 @@ impl StorageMigrationService {
|
||||
.await
|
||||
.is_ok();
|
||||
self.migration_readonly.store(false, Ordering::Relaxed);
|
||||
// Clear the shared progress snapshot so the server-status
|
||||
// header stops emitting on subsequent requests. Guard held
|
||||
// only for the assignment.
|
||||
{
|
||||
let mut guard = self
|
||||
.migration_progress
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
*guard = None;
|
||||
}
|
||||
|
||||
if !readonly_persisted {
|
||||
tracing::warn!(
|
||||
|
||||
@@ -687,13 +687,24 @@ pub fn create_api_routes(app_state: &Arc<AppState>) -> Router<Arc<AppState>> {
|
||||
// them on every overlapping request.
|
||||
router = router.route("/{*rest}", any(api_not_found));
|
||||
|
||||
// No per-router layers: the global `TraceLayer` + request-id stack in
|
||||
// `main.rs` wraps the whole app (this `/api` router is nested into it),
|
||||
// so a second `TraceLayer` here just double-wrapped every `/api`
|
||||
// request in a redundant span + response-future poll (benches/ROUND13.md
|
||||
// §H1). Compression is likewise the global layer's job — re-applying it
|
||||
// here (no predicate) would compress media downloads, burning CPU for
|
||||
// ~0 gain and stripping `Content-Length`.
|
||||
// Server-status header. Stamps `X-Server-Status` on every
|
||||
// response so the frontend's fetch wrapper can update a
|
||||
// reactive store — banner shows/hides without polling.
|
||||
// Sub-nanosecond on the hot path (single atomic load), a few
|
||||
// µs on the cold path (only during a running migration). See
|
||||
// `middleware::server_status`.
|
||||
let router = router.layer(axum::middleware::from_fn_with_state(
|
||||
app_state.clone(),
|
||||
crate::interfaces::middleware::server_status::server_status_middleware,
|
||||
));
|
||||
|
||||
// No per-router layers beyond that: the global `TraceLayer` + request-id
|
||||
// stack in `main.rs` wraps the whole app (this `/api` router is nested
|
||||
// into it), so a second `TraceLayer` here just double-wrapped every
|
||||
// `/api` request in a redundant span + response-future poll
|
||||
// (benches/ROUND13.md §H1). Compression is likewise the global layer's
|
||||
// job — re-applying it here (no predicate) would compress media
|
||||
// downloads, burning CPU for ~0 gain and stripping `Content-Length`.
|
||||
router
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ pub mod auth;
|
||||
pub mod csrf;
|
||||
pub mod locale;
|
||||
pub mod rate_limit;
|
||||
pub mod server_status;
|
||||
pub mod trace_span;
|
||||
pub mod trusted_proxy;
|
||||
pub mod user;
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Middleware that stamps `X-Server-Status` on every response.
|
||||
//!
|
||||
//! Consumed by the frontend `apiFetch` wrapper — every API round-trip
|
||||
//! carries the current server maintenance state back to the client
|
||||
//! (no polling, no dedicated endpoint). The banner in the app shell
|
||||
//! subscribes to a store the wrapper updates and shows/hides itself
|
||||
//! reactively. See `docs/plan/storage-multi-entry.md` §"Read-only mode"
|
||||
//! for the broader design.
|
||||
//!
|
||||
//! ## Cost model
|
||||
//!
|
||||
//! On the *hot path* (no migration running — the ~100% case in normal
|
||||
//! operation) this middleware does:
|
||||
//! 1. one `AtomicBool::load(Relaxed)` — sub-nanosecond;
|
||||
//! 2. an early return when `false`.
|
||||
//!
|
||||
//! No allocation, no lock, no formatting. Adds no measurable latency
|
||||
//! at any user count.
|
||||
//!
|
||||
//! On the *cold path* (migration in progress) this middleware does:
|
||||
//! 1. the atomic load above;
|
||||
//! 2. one `RwLock::read` (uncontended — writers are the migration
|
||||
//! handler, one per batch every ~100 blobs);
|
||||
//! 3. one small `serde_json::to_string` call on a 4-field struct
|
||||
//! (a few dozen bytes);
|
||||
//! 4. one header insertion.
|
||||
//!
|
||||
//! Total per-request work in this branch: microseconds.
|
||||
|
||||
use axum::extract::Request;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderValue;
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use crate::common::di::AppState;
|
||||
|
||||
/// Name of the response header the frontend reads. Kept short — an
|
||||
/// admin browser session may keep this header around in every open
|
||||
/// tab's dev-tools network view during a migration; the value is
|
||||
/// small JSON but the name should not add bloat.
|
||||
pub const SERVER_STATUS_HEADER: &str = "x-server-status";
|
||||
|
||||
/// Compact JSON shape written into the header. Fields are documented
|
||||
/// in `common::migration_progress::MigrationProgress`.
|
||||
///
|
||||
/// Kept internal so the wire format can evolve. Frontend treats the
|
||||
/// header as opaque JSON and pattern-matches on the fields it
|
||||
/// currently understands.
|
||||
#[derive(serde::Serialize)]
|
||||
struct HeaderPayload {
|
||||
readonly: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
migration: Option<MigrationHeader>,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct MigrationHeader {
|
||||
// `target` is owned here — the RwLock guard is released before
|
||||
// serialisation, so a borrowed slice wouldn't survive. Names
|
||||
// are small (`[a-z0-9_-]{1,32}`) so the copy is trivial.
|
||||
target: String,
|
||||
migrated: u64,
|
||||
total: u64,
|
||||
percent: u8,
|
||||
}
|
||||
|
||||
pub async fn server_status_middleware(
|
||||
State(state): State<Arc<AppState>>,
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
// Hot-path fast return. When no migration is running the flag is
|
||||
// false and there's nothing to emit — a bare atomic load and out.
|
||||
let readonly = state.migration_readonly.load(Ordering::Relaxed);
|
||||
let mut response = next.run(request).await;
|
||||
if !readonly {
|
||||
return response;
|
||||
}
|
||||
|
||||
// Cold path — build the payload from the shared progress
|
||||
// snapshot. If the snapshot is absent (readonly is true but the
|
||||
// handler hasn't seeded progress yet, or a restart-during-
|
||||
// migration scenario) we still emit `readonly: true` so the
|
||||
// banner shows — the frontend renders a "maintenance in progress"
|
||||
// message even when specific numbers aren't available.
|
||||
let payload = {
|
||||
let guard = state
|
||||
.migration_progress
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
HeaderPayload {
|
||||
readonly: true,
|
||||
migration: guard.as_ref().map(|p| MigrationHeader {
|
||||
target: p.target_name.clone(),
|
||||
migrated: p.migrated_blobs,
|
||||
total: p.total_blobs,
|
||||
percent: p.percent,
|
||||
}),
|
||||
}
|
||||
};
|
||||
|
||||
// `serde_json::to_string` on this 4-field struct is a few
|
||||
// dozen-byte allocation — negligible against the response body.
|
||||
// A serialize failure here would be a programming bug (all
|
||||
// fields are trivially serializable), so we degrade to a
|
||||
// minimal `readonly: true` string rather than skipping the
|
||||
// header entirely.
|
||||
let value =
|
||||
serde_json::to_string(&payload).unwrap_or_else(|_| r#"{"readonly":true}"#.to_string());
|
||||
if let Ok(header_value) = HeaderValue::from_str(&value) {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(SERVER_STATUS_HEADER, header_value);
|
||||
}
|
||||
response
|
||||
}
|
||||
Reference in New Issue
Block a user