feat(storage): add readonly during storage migration
This commit is contained in:
@@ -1810,6 +1810,7 @@ mod mount_authz_integration {
|
||||
Arc::new(FolderDbRepository::new(pool.clone())),
|
||||
Arc::new(FileBlobReadRepository::new_stub()),
|
||||
Arc::new(SubjectGroupPgRepository::new(pool.clone())),
|
||||
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -2377,7 +2378,13 @@ mod cascade_hook_integration_tests {
|
||||
folder_repo.clone(),
|
||||
));
|
||||
let group_repo = Arc::new(SubjectGroupPgRepository::new(pool.clone()));
|
||||
Arc::new(PgAclEngine::new(pool, folder_repo, file_repo, group_repo))
|
||||
Arc::new(PgAclEngine::new(
|
||||
pool,
|
||||
folder_repo,
|
||||
file_repo,
|
||||
group_repo,
|
||||
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
))
|
||||
}
|
||||
|
||||
/// Seed a file row under `folder_id`. `blob_hash` is just a string —
|
||||
|
||||
+142
-1
@@ -1487,11 +1487,34 @@ impl AppServiceFactory {
|
||||
let subject_group_repo = Arc::new(
|
||||
crate::infrastructure::repositories::pg::SubjectGroupPgRepository::new(pool.clone()),
|
||||
);
|
||||
// Migration-readonly atomic. Seeded from
|
||||
// `admin_settings.storage.migration_readonly` so the flag
|
||||
// survives restart (an operator won't see writes accidentally
|
||||
// re-enabled between a crash mid-migration and the retrigger).
|
||||
// Shared with the AuthZ engine so it can short-circuit write
|
||||
// permissions without a per-check DB round-trip. The boot
|
||||
// clear rule (§Read-only mode) runs after this seeding, after
|
||||
// the boot recovery sweep — enough for the runtime state
|
||||
// machine to decide whether to keep or clear.
|
||||
let migration_readonly = Arc::new(std::sync::atomic::AtomicBool::new(
|
||||
crate::infrastructure::services::entry_backend::load_migration_readonly(&pool).await,
|
||||
));
|
||||
if migration_readonly.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
tracing::warn!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "storage.migration_readonly.loaded_true_at_boot",
|
||||
"Server booted with migration_readonly=true — writes will be refused by AuthZ \
|
||||
until the flag is cleared (either by the boot-clear rule or via the admin \
|
||||
storage tab)."
|
||||
);
|
||||
}
|
||||
|
||||
let authorization = build_authorization_engine(
|
||||
pool.clone(),
|
||||
repos.folder_repository.clone(),
|
||||
repos.file_read_repository.clone(),
|
||||
subject_group_repo.clone(),
|
||||
migration_readonly.clone(),
|
||||
);
|
||||
|
||||
// Recent service + recording hook are built up-front so the
|
||||
@@ -1979,6 +2002,7 @@ impl AppServiceFactory {
|
||||
webdav_dead_props:
|
||||
crate::infrastructure::services::webdav_dead_property_store::create_dead_property_store(pool.clone()),
|
||||
authorization: authorization.clone(),
|
||||
migration_readonly: migration_readonly.clone(),
|
||||
drive_repo: drive_repo.clone(),
|
||||
drive_management_service: Arc::new(
|
||||
crate::application::services::drive_management_service::DriveManagementService::new(
|
||||
@@ -2195,6 +2219,7 @@ impl AppServiceFactory {
|
||||
app_state.core.active_backend_name.clone(),
|
||||
app_state.core.config.storage_entries.clone(),
|
||||
self.storage_path.clone(),
|
||||
app_state.migration_readonly.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&app_state.core.job_registry, &job_store_provider_dyn)
|
||||
@@ -2393,6 +2418,105 @@ impl AppServiceFactory {
|
||||
),
|
||||
}
|
||||
|
||||
// Migration-readonly boot-clear rule. See
|
||||
// `docs/plan/storage-multi-entry.md` §"Read-only mode".
|
||||
//
|
||||
// If the flag was set true at boot AND no storage_migration
|
||||
// run is currently non-terminal AND active_backend_name
|
||||
// matches the entry the app actually booted onto — that means
|
||||
// the cutover completed on a prior boot (the run reached
|
||||
// Completed, the pointer flipped, the operator restarted).
|
||||
// Safe to clear now: no in-flight migration means no one
|
||||
// still needs writes-off, and matching active_backend_name
|
||||
// means we're already on the target the run was pointing at.
|
||||
//
|
||||
// If ANY of those conditions fails (flag was false at boot;
|
||||
// there's still a Paused/Running/CancelRequested run in the
|
||||
// way; active doesn't match booted — mismatch means someone
|
||||
// manually edited the pointer while readonly was on) we
|
||||
// leave the flag alone. Operator has to decide.
|
||||
if app_state
|
||||
.migration_readonly
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
{
|
||||
use crate::infrastructure::services::storage_migration_service::STORAGE_MIGRATION_JOB_NAME;
|
||||
let has_in_flight = match app_state
|
||||
.core
|
||||
.job_store_provider
|
||||
.list_runs(STORAGE_MIGRATION_JOB_NAME, 5)
|
||||
.await
|
||||
{
|
||||
Ok(runs) => runs.iter().any(|r| {
|
||||
matches!(
|
||||
r.status,
|
||||
crate::infrastructure::scheduler::RunStatus::Running
|
||||
| crate::infrastructure::scheduler::RunStatus::Paused
|
||||
| crate::infrastructure::scheduler::RunStatus::CancelRequested
|
||||
)
|
||||
}),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "storage.migration_readonly.clear_check_failed",
|
||||
error = %e,
|
||||
"failed to list storage_migration runs during readonly-clear check; \
|
||||
leaving migration_readonly flag as-is"
|
||||
);
|
||||
// Play it safe: assume in-flight to avoid clearing prematurely.
|
||||
true
|
||||
}
|
||||
};
|
||||
|
||||
// Look up the DB pointer to compare against the booted
|
||||
// active_backend_name. Absence (Unset) is treated as "no
|
||||
// mismatch to complain about" — the boot fallback already
|
||||
// picked the first entry.
|
||||
let db_active_matches = {
|
||||
use crate::infrastructure::services::entry_backend::{
|
||||
ActiveEntry, resolve_active_entry,
|
||||
};
|
||||
match resolve_active_entry(&pool, &app_state.core.config.storage_entries).await {
|
||||
Ok(ActiveEntry::Explicit(e)) => e.name == app_state.core.active_backend_name,
|
||||
Ok(ActiveEntry::Unset) => true,
|
||||
Err(_) => false,
|
||||
}
|
||||
};
|
||||
|
||||
if !has_in_flight && db_active_matches {
|
||||
use crate::infrastructure::services::entry_backend::persist_migration_readonly;
|
||||
match persist_migration_readonly(&pool, false).await {
|
||||
Ok(()) => {
|
||||
app_state
|
||||
.migration_readonly
|
||||
.store(false, std::sync::atomic::Ordering::Relaxed);
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "storage.migration_readonly.cleared_at_boot",
|
||||
active = %app_state.core.active_backend_name,
|
||||
"🧊 migration_readonly cleared at boot: no in-flight migration + \
|
||||
active_backend_name matches booted entry (cutover complete on prior boot)"
|
||||
);
|
||||
}
|
||||
Err(e) => tracing::warn!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "storage.migration_readonly.clear_persist_failed",
|
||||
error = %e,
|
||||
"cleared migration_readonly in memory would have been safe, but the DB \
|
||||
write failed — leaving the DB row alone; will re-check next boot"
|
||||
),
|
||||
}
|
||||
} else {
|
||||
tracing::info!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "storage.migration_readonly.retained_at_boot",
|
||||
has_in_flight = has_in_flight,
|
||||
db_active_matches = db_active_matches,
|
||||
"migration_readonly retained at boot (in-flight run and/or active-name \
|
||||
mismatch prevents auto-clear)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Start the periodic-job scheduler AFTER every native service has
|
||||
// finished registering its jobs on `core.job_registry`. Starting
|
||||
// it earlier would race the first tick against late registrations.
|
||||
@@ -2589,6 +2713,16 @@ pub struct AppState {
|
||||
/// an enum dispatcher or `Arc<dyn AuthorizationEngine>` (with
|
||||
/// `async_trait` boxing).
|
||||
pub authorization: Arc<crate::infrastructure::services::pg_acl_engine::PgAclEngine>,
|
||||
/// Global "server is in migration read-only mode" flag, shared
|
||||
/// with [`Self::authorization`] so it can short-circuit write
|
||||
/// permissions. Backed by
|
||||
/// `admin_settings.storage.migration_readonly` for restart
|
||||
/// survival. Slice 5's cutover state machine flips this atomic
|
||||
/// (via `Ordering::Relaxed`) and calls
|
||||
/// `entry_backend::persist_migration_readonly` to keep DB and
|
||||
/// memory in sync. See `docs/plan/storage-multi-entry.md`
|
||||
/// §"Read-only mode".
|
||||
pub migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||
/// Drive entity repository — `GET /api/drives`, the personal-drive
|
||||
/// lifecycle hook, and (post-D2) shared-drive creation flow all read
|
||||
/// through this. Backing table is `storage.drives`; membership is
|
||||
@@ -2731,6 +2865,7 @@ fn build_authorization_engine(
|
||||
crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlobReadRepository,
|
||||
>,
|
||||
group_repo: Arc<crate::infrastructure::repositories::pg::SubjectGroupPgRepository>,
|
||||
migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||
) -> Arc<crate::infrastructure::services::pg_acl_engine::PgAclEngine> {
|
||||
use crate::infrastructure::services::pg_acl_engine::PgAclEngine;
|
||||
|
||||
@@ -2742,7 +2877,13 @@ fn build_authorization_engine(
|
||||
"OXICLOUD_AUTHZ_ENGINE={other:?} is not yet supported. Only 'postgres' is implemented; leave the variable unset to use the default."
|
||||
);
|
||||
}
|
||||
Arc::new(PgAclEngine::new(pool, folder_repo, file_repo, group_repo))
|
||||
Arc::new(PgAclEngine::new(
|
||||
pool,
|
||||
folder_repo,
|
||||
file_repo,
|
||||
group_repo,
|
||||
migration_readonly,
|
||||
))
|
||||
}
|
||||
|
||||
/// Pair returned by [`build_email_sender`] when wiring DI: the
|
||||
|
||||
@@ -34,6 +34,84 @@ use crate::common::config::{NamedStorageEntry, StorageBackendType};
|
||||
/// selection (see `docs/plan/storage-multi-entry.md` §"One DB row").
|
||||
pub const ACTIVE_BACKEND_NAME_KEY: &str = "storage.active_backend_name";
|
||||
|
||||
/// Key in `auth.admin_settings` that holds the persistent-across-restart
|
||||
/// migration-readonly flag. See
|
||||
/// `docs/plan/storage-multi-entry.md` §"Read-only mode reuses the
|
||||
/// existing AuthZ short-circuit". Value is `"true"` or `"false"`
|
||||
/// (plain text; the settings table stores strings).
|
||||
pub const MIGRATION_READONLY_KEY: &str = "storage.migration_readonly";
|
||||
|
||||
/// Read the persisted `migration_readonly` flag from `admin_settings`.
|
||||
/// Absent row / parse failure / DB error all resolve to `false` — the
|
||||
/// safer default when we can't determine the intent, since a false
|
||||
/// value only means "writes allowed by AuthZ" not "migration is
|
||||
/// running." Called once at boot to seed the in-memory `AtomicBool`.
|
||||
pub async fn load_migration_readonly(pool: &PgPool) -> bool {
|
||||
let row: Result<Option<(Option<String>,)>, sqlx::Error> =
|
||||
sqlx::query_as("SELECT value FROM auth.admin_settings WHERE key = $1")
|
||||
.bind(MIGRATION_READONLY_KEY)
|
||||
.fetch_optional(pool)
|
||||
.await;
|
||||
match row {
|
||||
Ok(Some((Some(v),))) => matches!(v.to_lowercase().as_str(), "true" | "1"),
|
||||
Ok(_) => false,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "storage.migration_readonly.load_failed",
|
||||
error = %e,
|
||||
"failed to read {MIGRATION_READONLY_KEY} at boot; defaulting to false"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Persist the `migration_readonly` flag. Idempotent — upserts the
|
||||
/// `admin_settings` row. Called by the cutover state machine (slice 5)
|
||||
/// when a migration starts (set true) or completes cleanly across a
|
||||
/// restart (set false via the boot clear rule). Handler / trigger
|
||||
/// callers should also update the in-memory `AtomicBool` alongside
|
||||
/// this call to keep the two in sync.
|
||||
pub async fn persist_migration_readonly(pool: &PgPool, value: bool) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO auth.admin_settings (key, value, category, is_secret)
|
||||
VALUES ($1, $2, 'storage', FALSE)
|
||||
ON CONFLICT (key)
|
||||
DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()
|
||||
"#,
|
||||
)
|
||||
.bind(MIGRATION_READONLY_KEY)
|
||||
.bind(if value { "true" } else { "false" })
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist the `active_backend_name` pointer. Called by the migration
|
||||
/// handler on `RunOutcome::Completed` to flip the runtime backend to
|
||||
/// the just-migrated target entry. The next boot reads this via
|
||||
/// `resolve_active_entry` and picks the new entry for the LIVE
|
||||
/// backend; before the restart the process is still on the OLD
|
||||
/// backend (that's what the `migration_readonly` gate is protecting).
|
||||
/// Idempotent UPSERT.
|
||||
pub async fn persist_active_backend_name(pool: &PgPool, name: &str) -> Result<(), sqlx::Error> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO auth.admin_settings (key, value, category, is_secret)
|
||||
VALUES ($1, $2, 'storage', FALSE)
|
||||
ON CONFLICT (key)
|
||||
DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()
|
||||
"#,
|
||||
)
|
||||
.bind(ACTIVE_BACKEND_NAME_KEY)
|
||||
.bind(name)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Result of [`resolve_active_entry`].
|
||||
pub enum ActiveEntry<'a> {
|
||||
/// DB has an `active_backend_name` set AND that name matches an
|
||||
|
||||
@@ -253,6 +253,18 @@ pub struct PgAclEngine {
|
||||
/// Total parent-resolution queries actually issued (point + batches) —
|
||||
/// exposed via [`Self::parent_query_count`] for benches/operators.
|
||||
parent_queries: Arc<AtomicU64>,
|
||||
/// Global "server is in migration read-only mode" flag. When
|
||||
/// `true`, `check_inner` short-circuits every write-adjacent
|
||||
/// permission (`Create`/`Update`/`Delete`/`Share`/`Comment`/`Manage`)
|
||||
/// with a `Denied` decision — same reason as the per-drive
|
||||
/// `read_only` gate below, but scoped to the whole process rather
|
||||
/// than a specific drive. Backed by
|
||||
/// `admin_settings.storage.migration_readonly` so it survives
|
||||
/// restart (see `docs/plan/storage-multi-entry.md` §"Read-only
|
||||
/// mode"). Shared as `Arc<AtomicBool>` with `AppState` so the
|
||||
/// cutover state machine (slice 5) can flip it without needing
|
||||
/// to reach into the engine.
|
||||
migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
/// One parked parent-resolution request: file id + reply slot. A dropped
|
||||
@@ -297,12 +309,14 @@ impl PgAclEngine {
|
||||
folder_repo: Arc<FolderDbRepository>,
|
||||
file_repo: Arc<FileBlobReadRepository>,
|
||||
group_repo: Arc<SubjectGroupPgRepository>,
|
||||
migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||
) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
folder_repo,
|
||||
file_repo,
|
||||
group_repo: Some(group_repo),
|
||||
migration_readonly,
|
||||
user_groups_cache: Cache::builder()
|
||||
.max_capacity(50_000)
|
||||
.time_to_live(Duration::from_secs(30))
|
||||
@@ -424,6 +438,7 @@ impl PgAclEngine {
|
||||
.build(),
|
||||
parent_batch: Arc::new(std::sync::Mutex::new(None)),
|
||||
parent_queries: Arc::new(AtomicU64::new(0)),
|
||||
migration_readonly: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1461,6 +1476,38 @@ impl PgAclEngine {
|
||||
resource: Resource,
|
||||
counters: &QueryCounters,
|
||||
) -> Result<bool, DomainError> {
|
||||
// Global migration-readonly short-circuit. Applies to every
|
||||
// resource type — no drive lookup, no per-resource state. When
|
||||
// the server is in migration read-only mode, every mutating
|
||||
// permission is refused with an audit line naming the specific
|
||||
// `migration_readonly` reason so operators filtering the audit
|
||||
// stream can distinguish it from per-drive freezes. Reads pass
|
||||
// (browsers, downloads, PROPFIND all keep working — same as the
|
||||
// per-drive gate). Admin operations don't reach `check_inner`
|
||||
// — they go through `admin_guard` middleware which bypasses
|
||||
// authz entirely, so the admin can still exit the mode, cancel
|
||||
// the migration, restart the server, etc.
|
||||
//
|
||||
// See `docs/plan/storage-multi-entry.md` §"Read-only mode".
|
||||
if Self::read_only_gate_applies(permission)
|
||||
&& self
|
||||
.migration_readonly
|
||||
.load(std::sync::atomic::Ordering::Relaxed)
|
||||
{
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "authz.denied",
|
||||
reason = "migration_readonly",
|
||||
subject_type = subject.type_str(),
|
||||
subject_id = %subject.id(),
|
||||
permission = permission.as_str(),
|
||||
resource_type = resource.type_str(),
|
||||
resource_id = %resource.id(),
|
||||
"🚧 mutation refused: server is in storage-migration read-only mode",
|
||||
);
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Drive-membership precheck for File/Folder. A role on the resource's
|
||||
// drive is the baseline floor (`drive.md §5`): the caller passes any
|
||||
// permission check the role bundle covers. Replaces the legacy
|
||||
|
||||
@@ -48,6 +48,7 @@
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures::StreamExt;
|
||||
@@ -60,7 +61,9 @@ use crate::infrastructure::scheduler::{
|
||||
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome,
|
||||
RunStatus, record_or_log,
|
||||
};
|
||||
use crate::infrastructure::services::entry_backend::build_entry_backend;
|
||||
use crate::infrastructure::services::entry_backend::{
|
||||
build_entry_backend, persist_active_backend_name, persist_migration_readonly,
|
||||
};
|
||||
|
||||
pub const STORAGE_MIGRATION_JOB_NAME: &str = "storage_migration";
|
||||
|
||||
@@ -98,15 +101,26 @@ pub struct StorageMigrationService {
|
||||
/// own `_ROOT_DIR`. Same fallback rule as boot
|
||||
/// (`build_entry_backend`).
|
||||
storage_path_fallback: PathBuf,
|
||||
/// Shared `AppState.migration_readonly` handle. Handler flips
|
||||
/// this atomic (and persists to DB) at run start once all
|
||||
/// guards pass, so writes across the whole app get refused by
|
||||
/// the AuthZ short-circuit for the duration of the copy. Kept
|
||||
/// ON when Completed — the boot-clear rule (slice 4) resets it
|
||||
/// on the next restart after cutover, so operators can't
|
||||
/// accidentally re-enable writes on the OLD backend while the
|
||||
/// pointer already says the NEW one is active.
|
||||
migration_readonly: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl StorageMigrationService {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
pool: Arc<PgPool>,
|
||||
source: Arc<dyn BlobStorageBackend>,
|
||||
active_backend_name: String,
|
||||
storage_entries: Vec<NamedStorageEntry>,
|
||||
storage_path_fallback: PathBuf,
|
||||
migration_readonly: Arc<AtomicBool>,
|
||||
) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
@@ -114,6 +128,7 @@ impl StorageMigrationService {
|
||||
active_backend_name,
|
||||
storage_entries,
|
||||
storage_path_fallback,
|
||||
migration_readonly,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -313,6 +328,37 @@ impl RecoverableJobHandler for StorageMigrationService {
|
||||
};
|
||||
}
|
||||
|
||||
// All guards passed. Engage server-wide read-only mode for
|
||||
// the duration of the copy so new writes can't create blobs
|
||||
// the migration walk has already stepped past. Both DB and
|
||||
// in-memory atomic get flipped in lock-step. Idempotent under
|
||||
// resume — the row is already `true` from the original open
|
||||
// (survived a restart via slice 4's boot seed), but rewriting
|
||||
// it doesn't hurt.
|
||||
//
|
||||
// A DB persist failure aborts before any copy — we won't
|
||||
// silently proceed with writes-allowed. If the atomic write
|
||||
// succeeded but DB failed we'd still have writes-off in this
|
||||
// process, but a restart mid-migration would lose it. Fail
|
||||
// early instead so operators see the actual DB problem.
|
||||
if let Err(e) = persist_migration_readonly(self.pool.as_ref(), true).await {
|
||||
return RunOutcome::Failed {
|
||||
message: format!(
|
||||
"engage migration_readonly (persist): {e} — refusing to copy without the \
|
||||
write freeze in place"
|
||||
),
|
||||
};
|
||||
}
|
||||
self.migration_readonly.store(true, Ordering::Relaxed);
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "storage.migration_readonly.engaged",
|
||||
run_id = %store.run_id(),
|
||||
target_name = %target_name,
|
||||
"🚧 migration_readonly engaged: writes across the whole app are refused until \
|
||||
cutover completes and the operator restarts"
|
||||
);
|
||||
|
||||
let source_kind = self.source.backend_type();
|
||||
let target_kind = target.backend_type();
|
||||
tracing::info!(
|
||||
@@ -403,17 +449,16 @@ impl RecoverableJobHandler for StorageMigrationService {
|
||||
};
|
||||
|
||||
if rows.is_empty() {
|
||||
tracing::info!(
|
||||
target: "oxicloud::migration",
|
||||
event = "storage_migration.completed",
|
||||
run_id = %store.run_id(),
|
||||
copied = copied_count,
|
||||
skipped = skipped_count,
|
||||
failed = failed_count,
|
||||
source_missing = source_missing_count,
|
||||
"storage_migration completed"
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
return self
|
||||
.finish_completed(
|
||||
store,
|
||||
&target_name,
|
||||
copied_count,
|
||||
skipped_count,
|
||||
failed_count,
|
||||
source_missing_count,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
for (hash, size) in &rows {
|
||||
@@ -544,22 +589,74 @@ impl RecoverableJobHandler for StorageMigrationService {
|
||||
}
|
||||
|
||||
if (rows.len() as i64) < BATCH_SIZE {
|
||||
tracing::info!(
|
||||
target: "oxicloud::migration",
|
||||
event = "storage_migration.completed",
|
||||
run_id = %store.run_id(),
|
||||
copied = copied_count,
|
||||
skipped = skipped_count,
|
||||
failed = failed_count,
|
||||
source_missing = source_missing_count,
|
||||
"storage_migration completed"
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
return self
|
||||
.finish_completed(
|
||||
store,
|
||||
&target_name,
|
||||
copied_count,
|
||||
skipped_count,
|
||||
failed_count,
|
||||
source_missing_count,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageMigrationService {
|
||||
/// Terminal successful path — reached from both Completed sites
|
||||
/// in the batch loop (empty-first-batch and short-batch). Flips
|
||||
/// the runtime `active_backend_name` pointer to the target entry
|
||||
/// so the NEXT boot picks it up. Leaves `migration_readonly` ON
|
||||
/// — the boot-clear rule (slice 4) drops it after the operator
|
||||
/// restart when no in-flight run remains AND the DB pointer
|
||||
/// matches the entry the app booted onto.
|
||||
///
|
||||
/// Pointer-write failure is FATAL to the outcome. Reporting
|
||||
/// `Completed` while the DB still says the old entry is active
|
||||
/// would strand the migrated bytes: the next boot would come up
|
||||
/// on the OLD backend (writes to old!), while the operator
|
||||
/// thinks cutover is done. `Failed` keeps the situation legible:
|
||||
/// admin sees the error, can retry the pointer write, then
|
||||
/// restart.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn finish_completed(
|
||||
&self,
|
||||
store: &dyn JobStore,
|
||||
target_name: &str,
|
||||
copied: u64,
|
||||
skipped: u64,
|
||||
failed: u64,
|
||||
source_missing: u64,
|
||||
) -> RunOutcome {
|
||||
if let Err(e) = persist_active_backend_name(self.pool.as_ref(), target_name).await {
|
||||
return RunOutcome::Failed {
|
||||
message: format!(
|
||||
"copy finished but writing active_backend_name = `{target_name}` to \
|
||||
admin_settings failed: {e}. Bytes are on the target; retrigger the run \
|
||||
once the DB is reachable and it will short-circuit on already-present \
|
||||
blobs and re-attempt the pointer flip."
|
||||
),
|
||||
};
|
||||
}
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "storage_migration.completed",
|
||||
run_id = %store.run_id(),
|
||||
active_backend_name = target_name,
|
||||
previous_active = %self.active_backend_name,
|
||||
copied = copied,
|
||||
skipped = skipped,
|
||||
failed = failed,
|
||||
source_missing = source_missing,
|
||||
"✅ storage_migration completed — active_backend_name = `{target_name}`. Restart the \
|
||||
server to switch the live backend (migration_readonly stays ON until then)."
|
||||
);
|
||||
RunOutcome::Completed
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical-storage identity string for a `NamedStorageEntry`. Two
|
||||
/// entries with the same identity point at the same physical
|
||||
/// location (same disk dir, same S3 bucket, same Azure container)
|
||||
|
||||
Reference in New Issue
Block a user