feat(storage): add readonly during storage migration
This commit is contained in:
@@ -186,6 +186,7 @@ fn fresh_engine(pool: &Arc<PgPool>) -> Arc<PgAclEngine> {
|
|||||||
folder_repo,
|
folder_repo,
|
||||||
file_repo,
|
file_repo,
|
||||||
group_repo,
|
group_repo,
|
||||||
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -176,6 +176,7 @@ fn fresh_engine(pool: &Arc<PgPool>) -> Arc<PgAclEngine> {
|
|||||||
folder_repo,
|
folder_repo,
|
||||||
file_repo,
|
file_repo,
|
||||||
group_repo,
|
group_repo,
|
||||||
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -769,6 +769,7 @@ fn wopi_engine(pool: &Arc<PgPool>) -> (Arc<PgAclEngine>, Arc<FileBlobReadReposit
|
|||||||
folder_repo,
|
folder_repo,
|
||||||
file_repo.clone(),
|
file_repo.clone(),
|
||||||
group_repo,
|
group_repo,
|
||||||
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
)),
|
)),
|
||||||
file_repo,
|
file_repo,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -233,6 +233,7 @@ fn fresh_engine(pool: &Arc<PgPool>) -> (Arc<PgAclEngine>, Arc<FileBlobReadReposi
|
|||||||
folder_repo,
|
folder_repo,
|
||||||
file_repo.clone(),
|
file_repo.clone(),
|
||||||
group_repo,
|
group_repo,
|
||||||
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
));
|
));
|
||||||
(engine, file_repo)
|
(engine, file_repo)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -237,6 +237,7 @@ fn fresh_engine(pool: &Arc<PgPool>) -> Arc<PgAclEngine> {
|
|||||||
folder_repo,
|
folder_repo,
|
||||||
file_repo,
|
file_repo,
|
||||||
group_repo,
|
group_repo,
|
||||||
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1810,6 +1810,7 @@ mod mount_authz_integration {
|
|||||||
Arc::new(FolderDbRepository::new(pool.clone())),
|
Arc::new(FolderDbRepository::new(pool.clone())),
|
||||||
Arc::new(FileBlobReadRepository::new_stub()),
|
Arc::new(FileBlobReadRepository::new_stub()),
|
||||||
Arc::new(SubjectGroupPgRepository::new(pool.clone())),
|
Arc::new(SubjectGroupPgRepository::new(pool.clone())),
|
||||||
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2377,7 +2378,13 @@ mod cascade_hook_integration_tests {
|
|||||||
folder_repo.clone(),
|
folder_repo.clone(),
|
||||||
));
|
));
|
||||||
let group_repo = Arc::new(SubjectGroupPgRepository::new(pool.clone()));
|
let group_repo = Arc::new(SubjectGroupPgRepository::new(pool.clone()));
|
||||||
Arc::new(PgAclEngine::new(pool, folder_repo, file_repo, group_repo))
|
Arc::new(PgAclEngine::new(
|
||||||
|
pool,
|
||||||
|
folder_repo,
|
||||||
|
file_repo,
|
||||||
|
group_repo,
|
||||||
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Seed a file row under `folder_id`. `blob_hash` is just a string —
|
/// Seed a file row under `folder_id`. `blob_hash` is just a string —
|
||||||
|
|||||||
+142
-1
@@ -1487,11 +1487,34 @@ impl AppServiceFactory {
|
|||||||
let subject_group_repo = Arc::new(
|
let subject_group_repo = Arc::new(
|
||||||
crate::infrastructure::repositories::pg::SubjectGroupPgRepository::new(pool.clone()),
|
crate::infrastructure::repositories::pg::SubjectGroupPgRepository::new(pool.clone()),
|
||||||
);
|
);
|
||||||
|
// Migration-readonly atomic. Seeded from
|
||||||
|
// `admin_settings.storage.migration_readonly` so the flag
|
||||||
|
// survives restart (an operator won't see writes accidentally
|
||||||
|
// re-enabled between a crash mid-migration and the retrigger).
|
||||||
|
// Shared with the AuthZ engine so it can short-circuit write
|
||||||
|
// permissions without a per-check DB round-trip. The boot
|
||||||
|
// clear rule (§Read-only mode) runs after this seeding, after
|
||||||
|
// the boot recovery sweep — enough for the runtime state
|
||||||
|
// machine to decide whether to keep or clear.
|
||||||
|
let migration_readonly = Arc::new(std::sync::atomic::AtomicBool::new(
|
||||||
|
crate::infrastructure::services::entry_backend::load_migration_readonly(&pool).await,
|
||||||
|
));
|
||||||
|
if migration_readonly.load(std::sync::atomic::Ordering::Relaxed) {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "oxicloud::scheduler",
|
||||||
|
event = "storage.migration_readonly.loaded_true_at_boot",
|
||||||
|
"Server booted with migration_readonly=true — writes will be refused by AuthZ \
|
||||||
|
until the flag is cleared (either by the boot-clear rule or via the admin \
|
||||||
|
storage tab)."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
let authorization = build_authorization_engine(
|
let authorization = build_authorization_engine(
|
||||||
pool.clone(),
|
pool.clone(),
|
||||||
repos.folder_repository.clone(),
|
repos.folder_repository.clone(),
|
||||||
repos.file_read_repository.clone(),
|
repos.file_read_repository.clone(),
|
||||||
subject_group_repo.clone(),
|
subject_group_repo.clone(),
|
||||||
|
migration_readonly.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Recent service + recording hook are built up-front so the
|
// Recent service + recording hook are built up-front so the
|
||||||
@@ -1979,6 +2002,7 @@ impl AppServiceFactory {
|
|||||||
webdav_dead_props:
|
webdav_dead_props:
|
||||||
crate::infrastructure::services::webdav_dead_property_store::create_dead_property_store(pool.clone()),
|
crate::infrastructure::services::webdav_dead_property_store::create_dead_property_store(pool.clone()),
|
||||||
authorization: authorization.clone(),
|
authorization: authorization.clone(),
|
||||||
|
migration_readonly: migration_readonly.clone(),
|
||||||
drive_repo: drive_repo.clone(),
|
drive_repo: drive_repo.clone(),
|
||||||
drive_management_service: Arc::new(
|
drive_management_service: Arc::new(
|
||||||
crate::application::services::drive_management_service::DriveManagementService::new(
|
crate::application::services::drive_management_service::DriveManagementService::new(
|
||||||
@@ -2195,6 +2219,7 @@ impl AppServiceFactory {
|
|||||||
app_state.core.active_backend_name.clone(),
|
app_state.core.active_backend_name.clone(),
|
||||||
app_state.core.config.storage_entries.clone(),
|
app_state.core.config.storage_entries.clone(),
|
||||||
self.storage_path.clone(),
|
self.storage_path.clone(),
|
||||||
|
app_state.migration_readonly.clone(),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.register_recoverable_job(&app_state.core.job_registry, &job_store_provider_dyn)
|
.register_recoverable_job(&app_state.core.job_registry, &job_store_provider_dyn)
|
||||||
@@ -2393,6 +2418,105 @@ impl AppServiceFactory {
|
|||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Migration-readonly boot-clear rule. See
|
||||||
|
// `docs/plan/storage-multi-entry.md` §"Read-only mode".
|
||||||
|
//
|
||||||
|
// If the flag was set true at boot AND no storage_migration
|
||||||
|
// run is currently non-terminal AND active_backend_name
|
||||||
|
// matches the entry the app actually booted onto — that means
|
||||||
|
// the cutover completed on a prior boot (the run reached
|
||||||
|
// Completed, the pointer flipped, the operator restarted).
|
||||||
|
// Safe to clear now: no in-flight migration means no one
|
||||||
|
// still needs writes-off, and matching active_backend_name
|
||||||
|
// means we're already on the target the run was pointing at.
|
||||||
|
//
|
||||||
|
// If ANY of those conditions fails (flag was false at boot;
|
||||||
|
// there's still a Paused/Running/CancelRequested run in the
|
||||||
|
// way; active doesn't match booted — mismatch means someone
|
||||||
|
// manually edited the pointer while readonly was on) we
|
||||||
|
// leave the flag alone. Operator has to decide.
|
||||||
|
if app_state
|
||||||
|
.migration_readonly
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed)
|
||||||
|
{
|
||||||
|
use crate::infrastructure::services::storage_migration_service::STORAGE_MIGRATION_JOB_NAME;
|
||||||
|
let has_in_flight = match app_state
|
||||||
|
.core
|
||||||
|
.job_store_provider
|
||||||
|
.list_runs(STORAGE_MIGRATION_JOB_NAME, 5)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(runs) => runs.iter().any(|r| {
|
||||||
|
matches!(
|
||||||
|
r.status,
|
||||||
|
crate::infrastructure::scheduler::RunStatus::Running
|
||||||
|
| crate::infrastructure::scheduler::RunStatus::Paused
|
||||||
|
| crate::infrastructure::scheduler::RunStatus::CancelRequested
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "oxicloud::scheduler",
|
||||||
|
event = "storage.migration_readonly.clear_check_failed",
|
||||||
|
error = %e,
|
||||||
|
"failed to list storage_migration runs during readonly-clear check; \
|
||||||
|
leaving migration_readonly flag as-is"
|
||||||
|
);
|
||||||
|
// Play it safe: assume in-flight to avoid clearing prematurely.
|
||||||
|
true
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Look up the DB pointer to compare against the booted
|
||||||
|
// active_backend_name. Absence (Unset) is treated as "no
|
||||||
|
// mismatch to complain about" — the boot fallback already
|
||||||
|
// picked the first entry.
|
||||||
|
let db_active_matches = {
|
||||||
|
use crate::infrastructure::services::entry_backend::{
|
||||||
|
ActiveEntry, resolve_active_entry,
|
||||||
|
};
|
||||||
|
match resolve_active_entry(&pool, &app_state.core.config.storage_entries).await {
|
||||||
|
Ok(ActiveEntry::Explicit(e)) => e.name == app_state.core.active_backend_name,
|
||||||
|
Ok(ActiveEntry::Unset) => true,
|
||||||
|
Err(_) => false,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if !has_in_flight && db_active_matches {
|
||||||
|
use crate::infrastructure::services::entry_backend::persist_migration_readonly;
|
||||||
|
match persist_migration_readonly(&pool, false).await {
|
||||||
|
Ok(()) => {
|
||||||
|
app_state
|
||||||
|
.migration_readonly
|
||||||
|
.store(false, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
tracing::info!(
|
||||||
|
target: "audit",
|
||||||
|
event = "storage.migration_readonly.cleared_at_boot",
|
||||||
|
active = %app_state.core.active_backend_name,
|
||||||
|
"🧊 migration_readonly cleared at boot: no in-flight migration + \
|
||||||
|
active_backend_name matches booted entry (cutover complete on prior boot)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(e) => tracing::warn!(
|
||||||
|
target: "oxicloud::scheduler",
|
||||||
|
event = "storage.migration_readonly.clear_persist_failed",
|
||||||
|
error = %e,
|
||||||
|
"cleared migration_readonly in memory would have been safe, but the DB \
|
||||||
|
write failed — leaving the DB row alone; will re-check next boot"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
tracing::info!(
|
||||||
|
target: "oxicloud::scheduler",
|
||||||
|
event = "storage.migration_readonly.retained_at_boot",
|
||||||
|
has_in_flight = has_in_flight,
|
||||||
|
db_active_matches = db_active_matches,
|
||||||
|
"migration_readonly retained at boot (in-flight run and/or active-name \
|
||||||
|
mismatch prevents auto-clear)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Start the periodic-job scheduler AFTER every native service has
|
// Start the periodic-job scheduler AFTER every native service has
|
||||||
// finished registering its jobs on `core.job_registry`. Starting
|
// finished registering its jobs on `core.job_registry`. Starting
|
||||||
// it earlier would race the first tick against late registrations.
|
// it earlier would race the first tick against late registrations.
|
||||||
@@ -2589,6 +2713,16 @@ pub struct AppState {
|
|||||||
/// an enum dispatcher or `Arc<dyn AuthorizationEngine>` (with
|
/// an enum dispatcher or `Arc<dyn AuthorizationEngine>` (with
|
||||||
/// `async_trait` boxing).
|
/// `async_trait` boxing).
|
||||||
pub authorization: Arc<crate::infrastructure::services::pg_acl_engine::PgAclEngine>,
|
pub authorization: Arc<crate::infrastructure::services::pg_acl_engine::PgAclEngine>,
|
||||||
|
/// Global "server is in migration read-only mode" flag, shared
|
||||||
|
/// with [`Self::authorization`] so it can short-circuit write
|
||||||
|
/// permissions. Backed by
|
||||||
|
/// `admin_settings.storage.migration_readonly` for restart
|
||||||
|
/// survival. Slice 5's cutover state machine flips this atomic
|
||||||
|
/// (via `Ordering::Relaxed`) and calls
|
||||||
|
/// `entry_backend::persist_migration_readonly` to keep DB and
|
||||||
|
/// memory in sync. See `docs/plan/storage-multi-entry.md`
|
||||||
|
/// §"Read-only mode".
|
||||||
|
pub migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||||
/// Drive entity repository — `GET /api/drives`, the personal-drive
|
/// Drive entity repository — `GET /api/drives`, the personal-drive
|
||||||
/// lifecycle hook, and (post-D2) shared-drive creation flow all read
|
/// lifecycle hook, and (post-D2) shared-drive creation flow all read
|
||||||
/// through this. Backing table is `storage.drives`; membership is
|
/// through this. Backing table is `storage.drives`; membership is
|
||||||
@@ -2731,6 +2865,7 @@ fn build_authorization_engine(
|
|||||||
crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlobReadRepository,
|
crate::infrastructure::repositories::pg::file_blob_read_repository::FileBlobReadRepository,
|
||||||
>,
|
>,
|
||||||
group_repo: Arc<crate::infrastructure::repositories::pg::SubjectGroupPgRepository>,
|
group_repo: Arc<crate::infrastructure::repositories::pg::SubjectGroupPgRepository>,
|
||||||
|
migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||||
) -> Arc<crate::infrastructure::services::pg_acl_engine::PgAclEngine> {
|
) -> Arc<crate::infrastructure::services::pg_acl_engine::PgAclEngine> {
|
||||||
use crate::infrastructure::services::pg_acl_engine::PgAclEngine;
|
use crate::infrastructure::services::pg_acl_engine::PgAclEngine;
|
||||||
|
|
||||||
@@ -2742,7 +2877,13 @@ fn build_authorization_engine(
|
|||||||
"OXICLOUD_AUTHZ_ENGINE={other:?} is not yet supported. Only 'postgres' is implemented; leave the variable unset to use the default."
|
"OXICLOUD_AUTHZ_ENGINE={other:?} is not yet supported. Only 'postgres' is implemented; leave the variable unset to use the default."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Arc::new(PgAclEngine::new(pool, folder_repo, file_repo, group_repo))
|
Arc::new(PgAclEngine::new(
|
||||||
|
pool,
|
||||||
|
folder_repo,
|
||||||
|
file_repo,
|
||||||
|
group_repo,
|
||||||
|
migration_readonly,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Pair returned by [`build_email_sender`] when wiring DI: the
|
/// Pair returned by [`build_email_sender`] when wiring DI: the
|
||||||
|
|||||||
@@ -34,6 +34,84 @@ use crate::common::config::{NamedStorageEntry, StorageBackendType};
|
|||||||
/// selection (see `docs/plan/storage-multi-entry.md` §"One DB row").
|
/// selection (see `docs/plan/storage-multi-entry.md` §"One DB row").
|
||||||
pub const ACTIVE_BACKEND_NAME_KEY: &str = "storage.active_backend_name";
|
pub const ACTIVE_BACKEND_NAME_KEY: &str = "storage.active_backend_name";
|
||||||
|
|
||||||
|
/// Key in `auth.admin_settings` that holds the persistent-across-restart
|
||||||
|
/// migration-readonly flag. See
|
||||||
|
/// `docs/plan/storage-multi-entry.md` §"Read-only mode reuses the
|
||||||
|
/// existing AuthZ short-circuit". Value is `"true"` or `"false"`
|
||||||
|
/// (plain text; the settings table stores strings).
|
||||||
|
pub const MIGRATION_READONLY_KEY: &str = "storage.migration_readonly";
|
||||||
|
|
||||||
|
/// Read the persisted `migration_readonly` flag from `admin_settings`.
|
||||||
|
/// Absent row / parse failure / DB error all resolve to `false` — the
|
||||||
|
/// safer default when we can't determine the intent, since a false
|
||||||
|
/// value only means "writes allowed by AuthZ" not "migration is
|
||||||
|
/// running." Called once at boot to seed the in-memory `AtomicBool`.
|
||||||
|
pub async fn load_migration_readonly(pool: &PgPool) -> bool {
|
||||||
|
let row: Result<Option<(Option<String>,)>, sqlx::Error> =
|
||||||
|
sqlx::query_as("SELECT value FROM auth.admin_settings WHERE key = $1")
|
||||||
|
.bind(MIGRATION_READONLY_KEY)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await;
|
||||||
|
match row {
|
||||||
|
Ok(Some((Some(v),))) => matches!(v.to_lowercase().as_str(), "true" | "1"),
|
||||||
|
Ok(_) => false,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "oxicloud::scheduler",
|
||||||
|
event = "storage.migration_readonly.load_failed",
|
||||||
|
error = %e,
|
||||||
|
"failed to read {MIGRATION_READONLY_KEY} at boot; defaulting to false"
|
||||||
|
);
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist the `migration_readonly` flag. Idempotent — upserts the
|
||||||
|
/// `admin_settings` row. Called by the cutover state machine (slice 5)
|
||||||
|
/// when a migration starts (set true) or completes cleanly across a
|
||||||
|
/// restart (set false via the boot clear rule). Handler / trigger
|
||||||
|
/// callers should also update the in-memory `AtomicBool` alongside
|
||||||
|
/// this call to keep the two in sync.
|
||||||
|
pub async fn persist_migration_readonly(pool: &PgPool, value: bool) -> Result<(), sqlx::Error> {
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
INSERT INTO auth.admin_settings (key, value, category, is_secret)
|
||||||
|
VALUES ($1, $2, 'storage', FALSE)
|
||||||
|
ON CONFLICT (key)
|
||||||
|
DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(MIGRATION_READONLY_KEY)
|
||||||
|
.bind(if value { "true" } else { "false" })
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist the `active_backend_name` pointer. Called by the migration
|
||||||
|
/// handler on `RunOutcome::Completed` to flip the runtime backend to
|
||||||
|
/// the just-migrated target entry. The next boot reads this via
|
||||||
|
/// `resolve_active_entry` and picks the new entry for the LIVE
|
||||||
|
/// backend; before the restart the process is still on the OLD
|
||||||
|
/// backend (that's what the `migration_readonly` gate is protecting).
|
||||||
|
/// Idempotent UPSERT.
|
||||||
|
pub async fn persist_active_backend_name(pool: &PgPool, name: &str) -> Result<(), sqlx::Error> {
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
INSERT INTO auth.admin_settings (key, value, category, is_secret)
|
||||||
|
VALUES ($1, $2, 'storage', FALSE)
|
||||||
|
ON CONFLICT (key)
|
||||||
|
DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(ACTIVE_BACKEND_NAME_KEY)
|
||||||
|
.bind(name)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Result of [`resolve_active_entry`].
|
/// Result of [`resolve_active_entry`].
|
||||||
pub enum ActiveEntry<'a> {
|
pub enum ActiveEntry<'a> {
|
||||||
/// DB has an `active_backend_name` set AND that name matches an
|
/// DB has an `active_backend_name` set AND that name matches an
|
||||||
|
|||||||
@@ -253,6 +253,18 @@ pub struct PgAclEngine {
|
|||||||
/// Total parent-resolution queries actually issued (point + batches) —
|
/// Total parent-resolution queries actually issued (point + batches) —
|
||||||
/// exposed via [`Self::parent_query_count`] for benches/operators.
|
/// exposed via [`Self::parent_query_count`] for benches/operators.
|
||||||
parent_queries: Arc<AtomicU64>,
|
parent_queries: Arc<AtomicU64>,
|
||||||
|
/// Global "server is in migration read-only mode" flag. When
|
||||||
|
/// `true`, `check_inner` short-circuits every write-adjacent
|
||||||
|
/// permission (`Create`/`Update`/`Delete`/`Share`/`Comment`/`Manage`)
|
||||||
|
/// with a `Denied` decision — same reason as the per-drive
|
||||||
|
/// `read_only` gate below, but scoped to the whole process rather
|
||||||
|
/// than a specific drive. Backed by
|
||||||
|
/// `admin_settings.storage.migration_readonly` so it survives
|
||||||
|
/// restart (see `docs/plan/storage-multi-entry.md` §"Read-only
|
||||||
|
/// mode"). Shared as `Arc<AtomicBool>` with `AppState` so the
|
||||||
|
/// cutover state machine (slice 5) can flip it without needing
|
||||||
|
/// to reach into the engine.
|
||||||
|
migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One parked parent-resolution request: file id + reply slot. A dropped
|
/// One parked parent-resolution request: file id + reply slot. A dropped
|
||||||
@@ -297,12 +309,14 @@ impl PgAclEngine {
|
|||||||
folder_repo: Arc<FolderDbRepository>,
|
folder_repo: Arc<FolderDbRepository>,
|
||||||
file_repo: Arc<FileBlobReadRepository>,
|
file_repo: Arc<FileBlobReadRepository>,
|
||||||
group_repo: Arc<SubjectGroupPgRepository>,
|
group_repo: Arc<SubjectGroupPgRepository>,
|
||||||
|
migration_readonly: Arc<std::sync::atomic::AtomicBool>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
pool,
|
pool,
|
||||||
folder_repo,
|
folder_repo,
|
||||||
file_repo,
|
file_repo,
|
||||||
group_repo: Some(group_repo),
|
group_repo: Some(group_repo),
|
||||||
|
migration_readonly,
|
||||||
user_groups_cache: Cache::builder()
|
user_groups_cache: Cache::builder()
|
||||||
.max_capacity(50_000)
|
.max_capacity(50_000)
|
||||||
.time_to_live(Duration::from_secs(30))
|
.time_to_live(Duration::from_secs(30))
|
||||||
@@ -424,6 +438,7 @@ impl PgAclEngine {
|
|||||||
.build(),
|
.build(),
|
||||||
parent_batch: Arc::new(std::sync::Mutex::new(None)),
|
parent_batch: Arc::new(std::sync::Mutex::new(None)),
|
||||||
parent_queries: Arc::new(AtomicU64::new(0)),
|
parent_queries: Arc::new(AtomicU64::new(0)),
|
||||||
|
migration_readonly: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1461,6 +1476,38 @@ impl PgAclEngine {
|
|||||||
resource: Resource,
|
resource: Resource,
|
||||||
counters: &QueryCounters,
|
counters: &QueryCounters,
|
||||||
) -> Result<bool, DomainError> {
|
) -> Result<bool, DomainError> {
|
||||||
|
// Global migration-readonly short-circuit. Applies to every
|
||||||
|
// resource type — no drive lookup, no per-resource state. When
|
||||||
|
// the server is in migration read-only mode, every mutating
|
||||||
|
// permission is refused with an audit line naming the specific
|
||||||
|
// `migration_readonly` reason so operators filtering the audit
|
||||||
|
// stream can distinguish it from per-drive freezes. Reads pass
|
||||||
|
// (browsers, downloads, PROPFIND all keep working — same as the
|
||||||
|
// per-drive gate). Admin operations don't reach `check_inner`
|
||||||
|
// — they go through `admin_guard` middleware which bypasses
|
||||||
|
// authz entirely, so the admin can still exit the mode, cancel
|
||||||
|
// the migration, restart the server, etc.
|
||||||
|
//
|
||||||
|
// See `docs/plan/storage-multi-entry.md` §"Read-only mode".
|
||||||
|
if Self::read_only_gate_applies(permission)
|
||||||
|
&& self
|
||||||
|
.migration_readonly
|
||||||
|
.load(std::sync::atomic::Ordering::Relaxed)
|
||||||
|
{
|
||||||
|
tracing::info!(
|
||||||
|
target: "audit",
|
||||||
|
event = "authz.denied",
|
||||||
|
reason = "migration_readonly",
|
||||||
|
subject_type = subject.type_str(),
|
||||||
|
subject_id = %subject.id(),
|
||||||
|
permission = permission.as_str(),
|
||||||
|
resource_type = resource.type_str(),
|
||||||
|
resource_id = %resource.id(),
|
||||||
|
"🚧 mutation refused: server is in storage-migration read-only mode",
|
||||||
|
);
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
// Drive-membership precheck for File/Folder. A role on the resource's
|
// Drive-membership precheck for File/Folder. A role on the resource's
|
||||||
// drive is the baseline floor (`drive.md §5`): the caller passes any
|
// drive is the baseline floor (`drive.md §5`): the caller passes any
|
||||||
// permission check the role bundle covers. Replaces the legacy
|
// permission check the role bundle covers. Replaces the legacy
|
||||||
|
|||||||
@@ -48,6 +48,7 @@
|
|||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
@@ -60,7 +61,9 @@ use crate::infrastructure::scheduler::{
|
|||||||
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome,
|
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome,
|
||||||
RunStatus, record_or_log,
|
RunStatus, record_or_log,
|
||||||
};
|
};
|
||||||
use crate::infrastructure::services::entry_backend::build_entry_backend;
|
use crate::infrastructure::services::entry_backend::{
|
||||||
|
build_entry_backend, persist_active_backend_name, persist_migration_readonly,
|
||||||
|
};
|
||||||
|
|
||||||
pub const STORAGE_MIGRATION_JOB_NAME: &str = "storage_migration";
|
pub const STORAGE_MIGRATION_JOB_NAME: &str = "storage_migration";
|
||||||
|
|
||||||
@@ -98,15 +101,26 @@ pub struct StorageMigrationService {
|
|||||||
/// own `_ROOT_DIR`. Same fallback rule as boot
|
/// own `_ROOT_DIR`. Same fallback rule as boot
|
||||||
/// (`build_entry_backend`).
|
/// (`build_entry_backend`).
|
||||||
storage_path_fallback: PathBuf,
|
storage_path_fallback: PathBuf,
|
||||||
|
/// Shared `AppState.migration_readonly` handle. Handler flips
|
||||||
|
/// this atomic (and persists to DB) at run start once all
|
||||||
|
/// guards pass, so writes across the whole app get refused by
|
||||||
|
/// the AuthZ short-circuit for the duration of the copy. Kept
|
||||||
|
/// ON when Completed — the boot-clear rule (slice 4) resets it
|
||||||
|
/// on the next restart after cutover, so operators can't
|
||||||
|
/// accidentally re-enable writes on the OLD backend while the
|
||||||
|
/// pointer already says the NEW one is active.
|
||||||
|
migration_readonly: Arc<AtomicBool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl StorageMigrationService {
|
impl StorageMigrationService {
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub fn new(
|
pub fn new(
|
||||||
pool: Arc<PgPool>,
|
pool: Arc<PgPool>,
|
||||||
source: Arc<dyn BlobStorageBackend>,
|
source: Arc<dyn BlobStorageBackend>,
|
||||||
active_backend_name: String,
|
active_backend_name: String,
|
||||||
storage_entries: Vec<NamedStorageEntry>,
|
storage_entries: Vec<NamedStorageEntry>,
|
||||||
storage_path_fallback: PathBuf,
|
storage_path_fallback: PathBuf,
|
||||||
|
migration_readonly: Arc<AtomicBool>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
pool,
|
pool,
|
||||||
@@ -114,6 +128,7 @@ impl StorageMigrationService {
|
|||||||
active_backend_name,
|
active_backend_name,
|
||||||
storage_entries,
|
storage_entries,
|
||||||
storage_path_fallback,
|
storage_path_fallback,
|
||||||
|
migration_readonly,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -313,6 +328,37 @@ impl RecoverableJobHandler for StorageMigrationService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// All guards passed. Engage server-wide read-only mode for
|
||||||
|
// the duration of the copy so new writes can't create blobs
|
||||||
|
// the migration walk has already stepped past. Both DB and
|
||||||
|
// in-memory atomic get flipped in lock-step. Idempotent under
|
||||||
|
// resume — the row is already `true` from the original open
|
||||||
|
// (survived a restart via slice 4's boot seed), but rewriting
|
||||||
|
// it doesn't hurt.
|
||||||
|
//
|
||||||
|
// A DB persist failure aborts before any copy — we won't
|
||||||
|
// silently proceed with writes-allowed. If the atomic write
|
||||||
|
// succeeded but DB failed we'd still have writes-off in this
|
||||||
|
// process, but a restart mid-migration would lose it. Fail
|
||||||
|
// early instead so operators see the actual DB problem.
|
||||||
|
if let Err(e) = persist_migration_readonly(self.pool.as_ref(), true).await {
|
||||||
|
return RunOutcome::Failed {
|
||||||
|
message: format!(
|
||||||
|
"engage migration_readonly (persist): {e} — refusing to copy without the \
|
||||||
|
write freeze in place"
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
self.migration_readonly.store(true, Ordering::Relaxed);
|
||||||
|
tracing::info!(
|
||||||
|
target: "audit",
|
||||||
|
event = "storage.migration_readonly.engaged",
|
||||||
|
run_id = %store.run_id(),
|
||||||
|
target_name = %target_name,
|
||||||
|
"🚧 migration_readonly engaged: writes across the whole app are refused until \
|
||||||
|
cutover completes and the operator restarts"
|
||||||
|
);
|
||||||
|
|
||||||
let source_kind = self.source.backend_type();
|
let source_kind = self.source.backend_type();
|
||||||
let target_kind = target.backend_type();
|
let target_kind = target.backend_type();
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
@@ -403,17 +449,16 @@ impl RecoverableJobHandler for StorageMigrationService {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if rows.is_empty() {
|
if rows.is_empty() {
|
||||||
tracing::info!(
|
return self
|
||||||
target: "oxicloud::migration",
|
.finish_completed(
|
||||||
event = "storage_migration.completed",
|
store,
|
||||||
run_id = %store.run_id(),
|
&target_name,
|
||||||
copied = copied_count,
|
copied_count,
|
||||||
skipped = skipped_count,
|
skipped_count,
|
||||||
failed = failed_count,
|
failed_count,
|
||||||
source_missing = source_missing_count,
|
source_missing_count,
|
||||||
"storage_migration completed"
|
)
|
||||||
);
|
.await;
|
||||||
return RunOutcome::Completed;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (hash, size) in &rows {
|
for (hash, size) in &rows {
|
||||||
@@ -544,19 +589,71 @@ impl RecoverableJobHandler for StorageMigrationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (rows.len() as i64) < BATCH_SIZE {
|
if (rows.len() as i64) < BATCH_SIZE {
|
||||||
|
return self
|
||||||
|
.finish_completed(
|
||||||
|
store,
|
||||||
|
&target_name,
|
||||||
|
copied_count,
|
||||||
|
skipped_count,
|
||||||
|
failed_count,
|
||||||
|
source_missing_count,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StorageMigrationService {
|
||||||
|
/// Terminal successful path — reached from both Completed sites
|
||||||
|
/// in the batch loop (empty-first-batch and short-batch). Flips
|
||||||
|
/// the runtime `active_backend_name` pointer to the target entry
|
||||||
|
/// so the NEXT boot picks it up. Leaves `migration_readonly` ON
|
||||||
|
/// — the boot-clear rule (slice 4) drops it after the operator
|
||||||
|
/// restart when no in-flight run remains AND the DB pointer
|
||||||
|
/// matches the entry the app booted onto.
|
||||||
|
///
|
||||||
|
/// Pointer-write failure is FATAL to the outcome. Reporting
|
||||||
|
/// `Completed` while the DB still says the old entry is active
|
||||||
|
/// would strand the migrated bytes: the next boot would come up
|
||||||
|
/// on the OLD backend (writes to old!), while the operator
|
||||||
|
/// thinks cutover is done. `Failed` keeps the situation legible:
|
||||||
|
/// admin sees the error, can retry the pointer write, then
|
||||||
|
/// restart.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
async fn finish_completed(
|
||||||
|
&self,
|
||||||
|
store: &dyn JobStore,
|
||||||
|
target_name: &str,
|
||||||
|
copied: u64,
|
||||||
|
skipped: u64,
|
||||||
|
failed: u64,
|
||||||
|
source_missing: u64,
|
||||||
|
) -> RunOutcome {
|
||||||
|
if let Err(e) = persist_active_backend_name(self.pool.as_ref(), target_name).await {
|
||||||
|
return RunOutcome::Failed {
|
||||||
|
message: format!(
|
||||||
|
"copy finished but writing active_backend_name = `{target_name}` to \
|
||||||
|
admin_settings failed: {e}. Bytes are on the target; retrigger the run \
|
||||||
|
once the DB is reachable and it will short-circuit on already-present \
|
||||||
|
blobs and re-attempt the pointer flip."
|
||||||
|
),
|
||||||
|
};
|
||||||
|
}
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
target: "oxicloud::migration",
|
target: "audit",
|
||||||
event = "storage_migration.completed",
|
event = "storage_migration.completed",
|
||||||
run_id = %store.run_id(),
|
run_id = %store.run_id(),
|
||||||
copied = copied_count,
|
active_backend_name = target_name,
|
||||||
skipped = skipped_count,
|
previous_active = %self.active_backend_name,
|
||||||
failed = failed_count,
|
copied = copied,
|
||||||
source_missing = source_missing_count,
|
skipped = skipped,
|
||||||
"storage_migration completed"
|
failed = failed,
|
||||||
|
source_missing = source_missing,
|
||||||
|
"✅ storage_migration completed — active_backend_name = `{target_name}`. Restart the \
|
||||||
|
server to switch the live backend (migration_readonly stays ON until then)."
|
||||||
);
|
);
|
||||||
return RunOutcome::Completed;
|
RunOutcome::Completed
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user