fix(storage migration): do not flip storage backend on migration error
This commit is contained in:
@@ -787,6 +787,68 @@ impl StorageMigrationService {
|
|||||||
failed: u64,
|
failed: u64,
|
||||||
source_missing: u64,
|
source_missing: u64,
|
||||||
) -> RunOutcome {
|
) -> RunOutcome {
|
||||||
|
// ── Failure gate ───────────────────────────────────────────
|
||||||
|
// Refuse to flip the active backend if ANY blob failed to
|
||||||
|
// migrate. Flipping to a partial target strands live traffic
|
||||||
|
// on incomplete data — reads for the missing hashes would
|
||||||
|
// 404. Findings are already recorded per-blob in the batch
|
||||||
|
// loop; operator inspects, then either:
|
||||||
|
// - retries (walk short-circuits on already-present blobs,
|
||||||
|
// so the retry costs only the failed ones), OR
|
||||||
|
// - fixes the source, retries, OR
|
||||||
|
// - accepts the loss and manually flips via
|
||||||
|
// `oxicloud --select-storage <target>`.
|
||||||
|
//
|
||||||
|
// Readonly is cleared either way — the source is still the
|
||||||
|
// active backend, and users shouldn't be locked out because
|
||||||
|
// of a partial run. Progress snapshot cleared too so the
|
||||||
|
// header middleware stops emitting.
|
||||||
|
if failed > 0 {
|
||||||
|
let readonly_persisted = persist_migration_readonly(self.pool.as_ref(), false)
|
||||||
|
.await
|
||||||
|
.is_ok();
|
||||||
|
self.migration_readonly.store(false, Ordering::Relaxed);
|
||||||
|
{
|
||||||
|
let mut guard = self
|
||||||
|
.migration_progress
|
||||||
|
.write()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
*guard = None;
|
||||||
|
}
|
||||||
|
if !readonly_persisted {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "oxicloud::migration",
|
||||||
|
event = "storage_migration.readonly_clear_persist_failed",
|
||||||
|
run_id = %store.run_id(),
|
||||||
|
"cleared migration_readonly in memory (writes allowed against source) but the \
|
||||||
|
DB persist failed. Boot-clear rule will fix on next restart."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
tracing::info!(
|
||||||
|
target: "audit",
|
||||||
|
event = "storage_migration.aborted",
|
||||||
|
reason = "blobs_failed",
|
||||||
|
run_id = %store.run_id(),
|
||||||
|
active_backend_name = previous_active,
|
||||||
|
target_name = target_name,
|
||||||
|
copied = copied,
|
||||||
|
skipped = skipped,
|
||||||
|
failed = failed,
|
||||||
|
source_missing = source_missing,
|
||||||
|
"🛑 storage_migration aborted — {failed} blob(s) failed, active backend left at \
|
||||||
|
`{previous_active}`, readonly cleared. Inspect findings and retry, or accept \
|
||||||
|
the partial migration via `oxicloud --select-storage {target_name}`."
|
||||||
|
);
|
||||||
|
return RunOutcome::Failed {
|
||||||
|
message: format!(
|
||||||
|
"{failed} blob(s) failed to migrate — active backend NOT switched \
|
||||||
|
(still `{previous_active}`). Retry the run (short-circuits on already-copied \
|
||||||
|
blobs) or accept the partial migration manually via \
|
||||||
|
`oxicloud --select-storage {target_name}`."
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
// 1. DB pointer.
|
// 1. DB pointer.
|
||||||
if let Err(e) = persist_active_backend_name(self.pool.as_ref(), target_name).await {
|
if let Err(e) = persist_active_backend_name(self.pool.as_ref(), target_name).await {
|
||||||
return RunOutcome::Failed {
|
return RunOutcome::Failed {
|
||||||
|
|||||||
Reference in New Issue
Block a user