feat(storage): wire choice of storage
This commit is contained in:
@@ -143,10 +143,19 @@ pub struct DashboardStatsDto {
|
||||
// Storage Settings DTOs (Admin Panel)
|
||||
// ============================================================================
|
||||
|
||||
/// Current storage settings returned to admin UI (secrets masked)
|
||||
/// Current storage settings returned to admin UI (secrets masked).
|
||||
///
|
||||
/// Post-multi-entry (`docs/plan/storage-multi-entry.md`) this carries
|
||||
/// two shapes side-by-side: the legacy flat storage-config fields
|
||||
/// (for the pre-multi-entry admin UI, until slice 6 completes the
|
||||
/// form retirement), plus the new `entries` / `active_entry_name` /
|
||||
/// `migration_readonly` view the multi-entry UI drives its entries-list,
|
||||
/// migration-target-dropdown, and readonly-banner from.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct StorageSettingsDto {
|
||||
/// Active backend type: "local" or "s3"
|
||||
/// Active backend type: "local" or "s3". Legacy flat-config field
|
||||
/// — mirrors `entries[i where is_active].backend` for the active
|
||||
/// entry when multi-entry is in use.
|
||||
pub backend: String,
|
||||
pub s3_endpoint_url: Option<String>,
|
||||
pub s3_bucket: Option<String>,
|
||||
@@ -163,6 +172,47 @@ pub struct StorageSettingsDto {
|
||||
pub total_blobs: u64,
|
||||
pub total_bytes_stored: u64,
|
||||
pub dedup_ratio: f64,
|
||||
// ── Multi-entry view (slice 6) ──
|
||||
/// All named storage entries declared in env. Empty when running
|
||||
/// in legacy single-backend mode (`OXICLOUD_STORAGE_ENTRIES`
|
||||
/// unset AND no legacy synthesis happened). Order matches
|
||||
/// `_ENTRIES`.
|
||||
pub entries: Vec<StorageEntrySummaryDto>,
|
||||
/// Name of the entry the LIVE backend is currently bound to.
|
||||
/// Populated as the boot-selected name (per
|
||||
/// `CoreServices.active_backend_name`). Empty string for the
|
||||
/// zero-entries legacy path (`"legacy"` sentinel).
|
||||
pub active_entry_name: String,
|
||||
/// Global read-only flag — when true, all write-adjacent
|
||||
/// AuthZ checks refuse. Set by the migration handler at run
|
||||
/// start; cleared by the boot-clear rule after operator
|
||||
/// restart. Frontend renders a banner on the storage tab when
|
||||
/// true.
|
||||
pub migration_readonly: bool,
|
||||
}
|
||||
|
||||
/// Per-entry summary emitted in `StorageSettingsDto.entries`. Never
|
||||
/// carries credentials — those live in env vars only. `is_active`
|
||||
/// marks which entry the LIVE backend uses right now (matches
|
||||
/// `active_entry_name` on the parent DTO).
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct StorageEntrySummaryDto {
|
||||
pub name: String,
|
||||
/// Backend type — "local" / "s3" / "azure".
|
||||
pub backend: String,
|
||||
/// True for exactly one entry (the entry the LIVE backend is on).
|
||||
/// Frontend uses this to badge the active row and to exclude it
|
||||
/// from the migration-target dropdown.
|
||||
pub is_active: bool,
|
||||
/// True when the entry has a per-entry encryption key. UI shows
|
||||
/// a lock icon. Presence-only — the key bytes never leave the
|
||||
/// server.
|
||||
pub encryption_enabled: bool,
|
||||
/// Human-readable physical location hint, if the backend surfaces
|
||||
/// one (`root_dir` for Local, `bucket` for S3, `container` for
|
||||
/// Azure). Cosmetic — helps the admin distinguish two Local
|
||||
/// entries pointing at different disks.
|
||||
pub location_hint: Option<String>,
|
||||
}
|
||||
|
||||
/// Request body for saving storage settings from the admin panel
|
||||
@@ -280,12 +330,11 @@ pub struct StartMigrationDto {
|
||||
pub concurrency: Option<usize>,
|
||||
}
|
||||
|
||||
/// Request body (empty) for `POST /api/admin/storage/migration/verify`.
|
||||
#[derive(Debug, Serialize, Deserialize, ToSchema)]
|
||||
pub struct VerifyMigrationDto {
|
||||
/// Number of random blobs to sample-check (default: 100).
|
||||
pub sample_size: Option<usize>,
|
||||
}
|
||||
// VerifyMigrationDto retired in slice 7 of
|
||||
// docs/plan/storage-multi-entry.md — the corresponding endpoint's
|
||||
// sample-based check is superseded by
|
||||
// `blobs_consistency?storage=<name>`, a full walk that emits
|
||||
// structured findings per mismatch.
|
||||
|
||||
// ============================================================================
|
||||
// SMTP Settings DTOs (Admin Panel)
|
||||
|
||||
@@ -2,11 +2,16 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use crate::application::dtos::settings_dto::{
|
||||
SaveStorageSettingsDto, StorageSettingsDto, StorageTestResultDto, TestStorageConnectionDto,
|
||||
SaveStorageSettingsDto, StorageEntrySummaryDto, StorageSettingsDto, StorageTestResultDto,
|
||||
TestStorageConnectionDto,
|
||||
};
|
||||
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||
use crate::common::config::{S3StorageConfig, StorageBackendType, StorageConfig};
|
||||
use crate::common::config::{
|
||||
NamedStorageEntry, S3StorageConfig, StorageBackendType, StorageConfig,
|
||||
};
|
||||
use crate::common::errors::{DomainError, ErrorKind};
|
||||
use crate::domain::repositories::settings_repository::SettingsRepository;
|
||||
use crate::infrastructure::repositories::pg::SettingsPgRepository;
|
||||
@@ -22,18 +27,39 @@ pub struct StorageSettingsService {
|
||||
settings_repo: Arc<SettingsPgRepository>,
|
||||
env_storage_config: StorageConfig,
|
||||
dedup_service: Arc<DedupService>,
|
||||
/// Multi-entry snapshot from `AppConfig.storage_entries`. Populated
|
||||
/// at DI time; immutable per-process (env can only change on
|
||||
/// restart, per `docs/plan/storage-multi-entry.md`). Empty when
|
||||
/// running in the pre-multi-entry legacy path.
|
||||
storage_entries: Vec<NamedStorageEntry>,
|
||||
/// Name of the entry the LIVE backend is bound to (matches
|
||||
/// `CoreServices.active_backend_name`). Empty string / "legacy"
|
||||
/// for the zero-entries path.
|
||||
active_entry_name: String,
|
||||
/// Shared readonly flag — read into the admin DTO so the UI can
|
||||
/// render a "server in migration read-only mode" banner. Same
|
||||
/// atomic as `AppState.migration_readonly`; changes made by the
|
||||
/// migration handler are visible without a DB round-trip.
|
||||
migration_readonly: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl StorageSettingsService {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
settings_repo: Arc<SettingsPgRepository>,
|
||||
env_storage_config: StorageConfig,
|
||||
dedup_service: Arc<DedupService>,
|
||||
storage_entries: Vec<NamedStorageEntry>,
|
||||
active_entry_name: String,
|
||||
migration_readonly: Arc<AtomicBool>,
|
||||
) -> Self {
|
||||
Self {
|
||||
settings_repo,
|
||||
env_storage_config,
|
||||
dedup_service,
|
||||
storage_entries,
|
||||
active_entry_name,
|
||||
migration_readonly,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,6 +288,27 @@ impl StorageSettingsService {
|
||||
crate::common::config::StorageBackendType::Azure => "azure",
|
||||
};
|
||||
|
||||
// Project the multi-entry view. `is_active` is name-compared
|
||||
// against the boot-selected `active_entry_name` (matches
|
||||
// exactly one entry when we're in multi-entry mode; matches
|
||||
// nothing when running the zero-entries legacy path, which
|
||||
// is expected — the frontend hides the entries table then).
|
||||
let entries: Vec<StorageEntrySummaryDto> = self
|
||||
.storage_entries
|
||||
.iter()
|
||||
.map(|e| StorageEntrySummaryDto {
|
||||
name: e.name.clone(),
|
||||
backend: match e.backend {
|
||||
StorageBackendType::Local => "local".to_string(),
|
||||
StorageBackendType::S3 => "s3".to_string(),
|
||||
StorageBackendType::Azure => "azure".to_string(),
|
||||
},
|
||||
is_active: e.name == self.active_entry_name,
|
||||
encryption_enabled: e.encryption_key_base64.is_some(),
|
||||
location_hint: entry_location_hint(e),
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(StorageSettingsDto {
|
||||
backend: backend_str.to_string(),
|
||||
s3_endpoint_url: effective.s3.as_ref().and_then(|s| s.endpoint_url.clone()),
|
||||
@@ -275,6 +322,9 @@ impl StorageSettingsService {
|
||||
total_blobs: stats.total_blobs,
|
||||
total_bytes_stored: stats.total_bytes_stored,
|
||||
dedup_ratio: stats.dedup_ratio,
|
||||
entries,
|
||||
active_entry_name: self.active_entry_name.clone(),
|
||||
migration_readonly: self.migration_readonly.load(Ordering::Relaxed),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -667,3 +717,17 @@ async fn run_backend_roundtrip(
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Cosmetic human-readable identifier for an entry — the physical
|
||||
/// location piece an admin uses to disambiguate two entries of the
|
||||
/// same backend type. Never carries credentials. `None` when the
|
||||
/// entry doesn't have a natural short label (S3 without a bucket,
|
||||
/// which shouldn't happen because the parser rejects that shape at
|
||||
/// boot).
|
||||
fn entry_location_hint(entry: &NamedStorageEntry) -> Option<String> {
|
||||
match entry.backend {
|
||||
StorageBackendType::Local => entry.root_dir.clone(),
|
||||
StorageBackendType::S3 => entry.s3.as_ref().map(|s3| s3.bucket.clone()),
|
||||
StorageBackendType::Azure => entry.azure.as_ref().map(|az| az.container.clone()),
|
||||
}
|
||||
}
|
||||
|
||||
+13
-1
@@ -1435,6 +1435,8 @@ impl AppServiceFactory {
|
||||
crate::infrastructure::services::blobs_consistency_service::BlobsConsistencyCheck::new(
|
||||
maintenance_pool.clone(),
|
||||
core.blob_backend.clone(),
|
||||
core.config.storage_entries.clone(),
|
||||
self.storage_path.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&core.job_registry, &job_store_provider_dyn)
|
||||
@@ -1453,6 +1455,8 @@ impl AppServiceFactory {
|
||||
crate::infrastructure::services::backend_consistency_service::BackendConsistencyCheck::new(
|
||||
maintenance_pool.clone(),
|
||||
core.blob_backend.clone(),
|
||||
core.config.storage_entries.clone(),
|
||||
self.storage_path.clone(),
|
||||
),
|
||||
)
|
||||
.register_recoverable_job(&core.job_registry, &job_store_provider_dyn)
|
||||
@@ -2191,11 +2195,19 @@ impl AppServiceFactory {
|
||||
|
||||
app_state.admin_settings_service = Some(admin_svc.clone());
|
||||
|
||||
// 9b-1b. Wire storage settings service (reuses same settings_repo)
|
||||
// 9b-1b. Wire storage settings service (reuses same settings_repo).
|
||||
// Multi-entry view fields (entries + active_entry_name +
|
||||
// migration_readonly) are populated from the same sources
|
||||
// the migration handler and AuthZ engine read from — one
|
||||
// snapshot at DI, shared atomic for the readonly flag so
|
||||
// changes are visible without a DB round-trip.
|
||||
let storage_settings_svc = Arc::new(StorageSettingsService::new(
|
||||
settings_repo.clone(),
|
||||
self.config.storage.clone(),
|
||||
app_state.core.dedup_service.clone(),
|
||||
app_state.core.config.storage_entries.clone(),
|
||||
app_state.core.active_backend_name.clone(),
|
||||
app_state.migration_readonly.clone(),
|
||||
));
|
||||
app_state.storage_settings_service = Some(storage_settings_svc.clone());
|
||||
tracing::info!("Storage settings service initialized");
|
||||
|
||||
@@ -57,6 +57,12 @@ use crate::infrastructure::scheduler::{
|
||||
|
||||
pub const BACKEND_CONSISTENCY_JOB_NAME: &str = "backend_consistency";
|
||||
|
||||
/// Same `params` JSONB key `blobs_consistency` uses — kept identical
|
||||
/// so operators grepping run rows see the same convention across
|
||||
/// both storage-audit tenants.
|
||||
pub const PROBED_STORAGE_PARAM: &str =
|
||||
crate::infrastructure::services::blobs_consistency_service::PROBED_STORAGE_PARAM;
|
||||
|
||||
/// Batch size for backend enumeration + DB probe. 500 is enough to
|
||||
/// amortise the DB round-trip while keeping the cancel-poll cadence
|
||||
/// sub-second (each batch = one backend list + one DB probe + Rust
|
||||
@@ -78,12 +84,32 @@ const _MAX_EXAMPLES: usize = 5;
|
||||
|
||||
pub struct BackendConsistencyCheck {
|
||||
pool: Arc<PgPool>,
|
||||
/// Default backend to enumerate when `args.storage` is `None` —
|
||||
/// the live LIVE backend, injected at DI. `?storage=<name>`
|
||||
/// swaps in a fresh backend for the named entry (via
|
||||
/// [`build_entry_backend`]).
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
/// Snapshot of `AppConfig.storage_entries` for `?storage=<name>`
|
||||
/// resolution. Same rule blobs_consistency uses.
|
||||
storage_entries: Vec<crate::common::config::NamedStorageEntry>,
|
||||
/// `OXICLOUD_STORAGE_PATH` fallback for Local entries with no
|
||||
/// `_ROOT_DIR`. Same fallback rule as boot.
|
||||
storage_path_fallback: std::path::PathBuf,
|
||||
}
|
||||
|
||||
impl BackendConsistencyCheck {
|
||||
pub fn new(pool: Arc<PgPool>, backend: Arc<dyn BlobStorageBackend>) -> Self {
|
||||
Self { pool, backend }
|
||||
pub fn new(
|
||||
pool: Arc<PgPool>,
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
storage_entries: Vec<crate::common::config::NamedStorageEntry>,
|
||||
storage_path_fallback: std::path::PathBuf,
|
||||
) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
backend,
|
||||
storage_entries,
|
||||
storage_path_fallback,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn register_recoverable_job(
|
||||
@@ -138,9 +164,76 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
||||
async fn run_resumable(
|
||||
&self,
|
||||
store: &dyn JobStore,
|
||||
_args: &JobRunArgs,
|
||||
args: &JobRunArgs,
|
||||
resume_cursor: Option<Vec<u8>>,
|
||||
) -> RunOutcome {
|
||||
// Resolve the backend to probe. Mirrors the shape
|
||||
// `blobs_consistency` uses — Fresh + args.storage=Some stamps
|
||||
// probed_storage into params; Resumed reads it back so a
|
||||
// mid-audit restart re-uses the same target.
|
||||
let is_fresh = resume_cursor.is_none();
|
||||
let probed_storage: Option<String> = if is_fresh {
|
||||
let name = args.storage.clone();
|
||||
if let Some(n) = &name
|
||||
&& let Err(e) = store.set_string_param(PROBED_STORAGE_PARAM, n).await
|
||||
{
|
||||
return RunOutcome::Failed {
|
||||
message: format!("persist {PROBED_STORAGE_PARAM} to params: {e}"),
|
||||
};
|
||||
}
|
||||
name
|
||||
} else {
|
||||
match store.get_string_param(PROBED_STORAGE_PARAM).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("read {PROBED_STORAGE_PARAM} from params: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
};
|
||||
let backend: Arc<dyn BlobStorageBackend> = match &probed_storage {
|
||||
None => self.backend.clone(),
|
||||
Some(name) => match self.storage_entries.iter().find(|e| &e.name == name) {
|
||||
Some(entry) => crate::infrastructure::services::entry_backend::build_entry_backend(
|
||||
entry,
|
||||
&self.storage_path_fallback,
|
||||
),
|
||||
None => {
|
||||
let available = if self.storage_entries.is_empty() {
|
||||
"(none)".to_string()
|
||||
} else {
|
||||
self.storage_entries
|
||||
.iter()
|
||||
.map(|e| e.name.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
};
|
||||
return RunOutcome::Failed {
|
||||
message: format!(
|
||||
"storage entry `{name}` not found in OXICLOUD_STORAGE_ENTRIES. \
|
||||
Available: [{available}]"
|
||||
),
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
if let Err(e) = backend.initialize().await {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("probed backend init: {e}"),
|
||||
};
|
||||
}
|
||||
if let Some(name) = &probed_storage {
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "backend_consistency.probe_scoped",
|
||||
run_id = %store.run_id(),
|
||||
probed_storage = %name,
|
||||
"backend_consistency enumerating entry `{name}` (via ?storage=<name>) instead \
|
||||
of live backend"
|
||||
);
|
||||
}
|
||||
|
||||
// Cursor = opaque backend continuation token, UTF-8-encoded.
|
||||
// Each backend defines its own format (local = shard/hash,
|
||||
// S3 = ListObjectsV2 continuation token, Azure = list
|
||||
@@ -190,11 +283,7 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
||||
// splits canonical blobs (checked for orphan) from
|
||||
// "unknown" entries (sidecar files, foreign namespaces —
|
||||
// emitted as informational notices).
|
||||
let page = match self
|
||||
.backend
|
||||
.list_blob_hashes(cursor.clone(), BATCH_SIZE)
|
||||
.await
|
||||
{
|
||||
let page = match backend.list_blob_hashes(cursor.clone(), BATCH_SIZE).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
// Backend refuses / can't enumerate. First-batch
|
||||
@@ -219,7 +308,7 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
||||
"anomaly",
|
||||
None,
|
||||
serde_json::json!({
|
||||
"backend": self.backend.backend_type(),
|
||||
"backend": backend.backend_type(),
|
||||
"error": format!("{e}"),
|
||||
"note": "backend refused enumeration; no per-blob orphan probes attempted",
|
||||
}),
|
||||
@@ -229,7 +318,7 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
||||
target: "oxicloud::consistency",
|
||||
event = "backend_consistency.unenumerable",
|
||||
run_id = %store.run_id(),
|
||||
backend = self.backend.backend_type(),
|
||||
backend = backend.backend_type(),
|
||||
"backend refused enumeration (typical during migration or on backends without list support)"
|
||||
);
|
||||
return RunOutcome::Completed;
|
||||
@@ -265,7 +354,7 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
||||
serde_json::json!({
|
||||
"path": unknown.path,
|
||||
"mtime": unknown.mtime.map(|t| t.to_rfc3339()),
|
||||
"backend": self.backend.backend_type(),
|
||||
"backend": backend.backend_type(),
|
||||
"note": "non-canonical file in blob namespace (sidecar / wrong extension); not managed by dedup",
|
||||
}),
|
||||
)
|
||||
@@ -331,7 +420,7 @@ impl RecoverableJobHandler for BackendConsistencyCheck {
|
||||
serde_json::json!({
|
||||
"hash": entry.hash,
|
||||
"mtime": entry.mtime.map(|t| t.to_rfc3339()),
|
||||
"backend": self.backend.backend_type(),
|
||||
"backend": backend.backend_type(),
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
//! pointing at reaped chunks) — already covered by
|
||||
//! `files_consistency::chunk_missing`.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -54,13 +55,21 @@ use chrono::{DateTime, Duration, Utc};
|
||||
use sqlx::PgPool;
|
||||
|
||||
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||
use crate::common::config::NamedStorageEntry;
|
||||
use crate::infrastructure::scheduler::{
|
||||
JobRegistry, JobRunArgs, JobStore, JobStoreProvider, RecoverableJobHandler, RunOutcome,
|
||||
RunStatus, record_or_log,
|
||||
};
|
||||
use crate::infrastructure::services::entry_backend::build_entry_backend;
|
||||
|
||||
pub const BLOBS_CONSISTENCY_JOB_NAME: &str = "blobs_consistency";
|
||||
|
||||
/// `params` JSONB key under which the entry name being probed is
|
||||
/// stashed on a Fresh run (matches `TARGET_NAME_PARAM` on
|
||||
/// `storage_migration`). Resumed runs re-read it so a paused audit
|
||||
/// survives restart without the admin re-specifying the target.
|
||||
pub const PROBED_STORAGE_PARAM: &str = "probed_storage";
|
||||
|
||||
/// Rows per batch. Blobs are numerous (millions on a busy install)
|
||||
/// but per-row work is one indexed backend probe + one indexed SQL
|
||||
/// ref-count query. 200 balances cancel-poll cadence against
|
||||
@@ -82,12 +91,35 @@ const AFFECTED_FILES_SAMPLE: i64 = 5;
|
||||
|
||||
pub struct BlobsConsistencyCheck {
|
||||
pool: Arc<PgPool>,
|
||||
/// The default backend to probe when `args.storage` is `None` —
|
||||
/// the currently-active LIVE backend, injected at DI time. Runs
|
||||
/// with `?storage=<name>` build a fresh backend for the named
|
||||
/// entry instead (via [`build_entry_backend`]).
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
/// Snapshot of `AppConfig.storage_entries` used to resolve
|
||||
/// `args.storage` to a `NamedStorageEntry`. Empty for the
|
||||
/// legacy zero-entries path — `?storage=<name>` runs then
|
||||
/// fail-fast with a clear "no entries declared" message.
|
||||
storage_entries: Vec<NamedStorageEntry>,
|
||||
/// Ambient `AppConfig.storage_path` — used as the `root_dir`
|
||||
/// fallback for a Local target entry with no `_ROOT_DIR`. Same
|
||||
/// fallback rule the boot path uses.
|
||||
storage_path_fallback: PathBuf,
|
||||
}
|
||||
|
||||
impl BlobsConsistencyCheck {
|
||||
pub fn new(pool: Arc<PgPool>, backend: Arc<dyn BlobStorageBackend>) -> Self {
|
||||
Self { pool, backend }
|
||||
pub fn new(
|
||||
pool: Arc<PgPool>,
|
||||
backend: Arc<dyn BlobStorageBackend>,
|
||||
storage_entries: Vec<NamedStorageEntry>,
|
||||
storage_path_fallback: PathBuf,
|
||||
) -> Self {
|
||||
Self {
|
||||
pool,
|
||||
backend,
|
||||
storage_entries,
|
||||
storage_path_fallback,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn register_recoverable_job(
|
||||
@@ -147,6 +179,80 @@ impl RecoverableJobHandler for BlobsConsistencyCheck {
|
||||
args: &JobRunArgs,
|
||||
resume_cursor: Option<Vec<u8>>,
|
||||
) -> RunOutcome {
|
||||
// Resolve the backend to probe. Two paths, mirroring the
|
||||
// Fresh/Resumed split the storage_migration handler uses:
|
||||
//
|
||||
// * Fresh + args.storage=Some — probe that named entry
|
||||
// instead of the live backend. Stamp probed_storage in
|
||||
// params so a mid-audit restart resumes against the same
|
||||
// entry without re-input.
|
||||
// * Fresh + args.storage=None — probe the live backend
|
||||
// (today's default; audit of what the app is actually
|
||||
// using).
|
||||
// * Resumed — read probed_storage from params; None means
|
||||
// the original run was against the live backend.
|
||||
let is_fresh = resume_cursor.is_none();
|
||||
let probed_storage: Option<String> = if is_fresh {
|
||||
let name = args.storage.clone();
|
||||
if let Some(n) = &name
|
||||
&& let Err(e) = store.set_string_param(PROBED_STORAGE_PARAM, n).await
|
||||
{
|
||||
return RunOutcome::Failed {
|
||||
message: format!(
|
||||
"persist {PROBED_STORAGE_PARAM} to params: {e}"
|
||||
),
|
||||
};
|
||||
}
|
||||
name
|
||||
} else {
|
||||
match store.get_string_param(PROBED_STORAGE_PARAM).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("read {PROBED_STORAGE_PARAM} from params: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
};
|
||||
let backend: Arc<dyn BlobStorageBackend> = match &probed_storage {
|
||||
None => self.backend.clone(),
|
||||
Some(name) => match self.storage_entries.iter().find(|e| &e.name == name) {
|
||||
Some(entry) => build_entry_backend(entry, &self.storage_path_fallback),
|
||||
None => {
|
||||
let available = if self.storage_entries.is_empty() {
|
||||
"(none)".to_string()
|
||||
} else {
|
||||
self.storage_entries
|
||||
.iter()
|
||||
.map(|e| e.name.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
};
|
||||
return RunOutcome::Failed {
|
||||
message: format!(
|
||||
"storage entry `{name}` not found in OXICLOUD_STORAGE_ENTRIES. \
|
||||
Available: [{available}]"
|
||||
),
|
||||
};
|
||||
}
|
||||
},
|
||||
};
|
||||
if let Err(e) = backend.initialize().await {
|
||||
return RunOutcome::Failed {
|
||||
message: format!("probed backend init: {e}"),
|
||||
};
|
||||
}
|
||||
if let Some(name) = &probed_storage {
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "blobs_consistency.probe_scoped",
|
||||
run_id = %store.run_id(),
|
||||
probed_storage = %name,
|
||||
"blobs_consistency probing entry `{name}` (via ?storage=<name>) instead of \
|
||||
live backend"
|
||||
);
|
||||
}
|
||||
|
||||
// Cursor = the last-visited `hash` string, UTF-8-encoded. On
|
||||
// resume, we walk `WHERE hash > $cursor` in ASC order. First
|
||||
// batch: NULL cursor → start from the smallest hash.
|
||||
@@ -318,7 +424,7 @@ impl RecoverableJobHandler for BlobsConsistencyCheck {
|
||||
// error (log + skip): a transient S3 network blip
|
||||
// shouldn't produce a flood of false data_loss
|
||||
// findings.
|
||||
let exists = match self.backend.blob_exists(&row.hash).await {
|
||||
let exists = match backend.blob_exists(&row.hash).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
@@ -369,7 +475,7 @@ impl RecoverableJobHandler for BlobsConsistencyCheck {
|
||||
// avoid mistaking it for "the hash we expect to
|
||||
// see on disk (i.e. what will fix this)".
|
||||
if args.deep {
|
||||
match recompute_hash(self.backend.as_ref(), &row.hash).await {
|
||||
match recompute_hash(backend.as_ref(), &row.hash).await {
|
||||
Ok(computed_hash) if computed_hash == row.hash => {}
|
||||
Ok(computed_hash) => {
|
||||
finding_count += 1;
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::application::dtos::settings_dto::{
|
||||
AdminCreateUserDto, AdminResetPasswordDto, DashboardStatsDto, ListUsersQueryDto,
|
||||
MigrationStateDto, SaveOidcSettingsDto, SaveStorageSettingsDto, SendSmtpTestDto, SmtpInfoDto,
|
||||
SmtpTestResultDto, StartMigrationDto, TestOidcConnectionDto, TestStorageConnectionDto,
|
||||
UpdateUserActiveDto, UpdateUserQuotaDto, UpdateUserRoleDto, VerifyMigrationDto,
|
||||
UpdateUserActiveDto, UpdateUserQuotaDto, UpdateUserRoleDto,
|
||||
};
|
||||
use crate::application::dtos::user_dto::{AdminUserSummaryDto, UserDto};
|
||||
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
||||
@@ -85,7 +85,9 @@ pub fn admin_routes() -> Router<Arc<AppState>> {
|
||||
.route("/storage/migration/start", post(start_migration))
|
||||
.route("/storage/migration/pause", post(pause_migration))
|
||||
.route("/storage/migration/resume", post(resume_migration))
|
||||
.route("/storage/migration/verify", post(verify_migration))
|
||||
// NOTE: /storage/migration/verify retired in slice 7 (see the
|
||||
// comment near where `verify_migration` used to live). Use
|
||||
// `POST /api/admin/jobs/blobs_consistency/trigger?storage=<name>`.
|
||||
// Encryption key generation
|
||||
.route(
|
||||
"/settings/storage/generate-key",
|
||||
@@ -562,55 +564,15 @@ pub async fn resume_migration(
|
||||
trigger_storage_migration(state, None).await
|
||||
}
|
||||
|
||||
/// POST /api/admin/storage/migration/verify — post-migration integrity check.
|
||||
///
|
||||
/// Independent of the copy job: samples `sample_size` random blobs
|
||||
/// from `storage.blobs` and probes the currently-effective target
|
||||
/// backend for their existence + declared size. Passes iff no
|
||||
/// samples are missing and no sizes disagree.
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/admin/storage/migration/verify",
|
||||
responses(
|
||||
(status = 200, description = "Verification result"),
|
||||
(status = 401, description = "Unauthorized"),
|
||||
(status = 403, description = "Admin required"),
|
||||
(status = 500, description = "Verification failed")
|
||||
),
|
||||
security(("bearerAuth" = [])),
|
||||
tag = "admin"
|
||||
)]
|
||||
pub async fn verify_migration(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Json(dto): Json<VerifyMigrationDto>,
|
||||
) -> Result<impl IntoResponse, AppError> {
|
||||
let pool = state
|
||||
.db_pool
|
||||
.clone()
|
||||
.ok_or_else(|| AppError::internal_error("Database not available"))?;
|
||||
|
||||
let svc = state
|
||||
.storage_settings_service
|
||||
.as_ref()
|
||||
.ok_or_else(|| AppError::internal_error("Storage settings service not available"))?;
|
||||
|
||||
let target = svc
|
||||
.build_effective_backend()
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to build target backend: {}", e)))?;
|
||||
target
|
||||
.initialize()
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Target backend init failed: {}", e)))?;
|
||||
|
||||
let sample_size = dto.sample_size.unwrap_or(100).clamp(1, 1000);
|
||||
|
||||
let result = verify_backend_sample(target.as_ref(), pool.as_ref(), sample_size)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Verification failed: {}", e)))?;
|
||||
|
||||
Ok(Json(result))
|
||||
}
|
||||
// verify_migration endpoint retired (slice 7 of
|
||||
// docs/plan/storage-multi-entry.md). It was a sample-based sanity
|
||||
// check against the currently-effective target backend; superseded
|
||||
// by `POST /api/admin/jobs/blobs_consistency/trigger?storage=<name>`
|
||||
// which does a full walk against ANY named entry (not just the
|
||||
// migration target), records structured findings per mismatch, and
|
||||
// integrates with the standard runs / cancel / findings admin
|
||||
// surface. Frontend "Verify integrity" button removed in the same
|
||||
// slice.
|
||||
|
||||
/// Shared body for `start` / `resume` — both funnel through
|
||||
/// `run_or_resume` via `JobRegistry::trigger`. Detaches into a
|
||||
@@ -652,73 +614,6 @@ async fn trigger_storage_migration(
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// Verify a random sample of blobs against the given target backend.
|
||||
/// Inlined from the retired `migration_job::verify_migration` — same
|
||||
/// query, same result shape; the recoverable-run engine has no reason
|
||||
/// to own an integrity check.
|
||||
async fn verify_backend_sample(
|
||||
target: &dyn crate::application::ports::blob_storage_ports::BlobStorageBackend,
|
||||
pool: &sqlx::PgPool,
|
||||
sample_size: usize,
|
||||
) -> Result<MigrationVerifyResult, crate::common::errors::DomainError> {
|
||||
use crate::common::errors::DomainError;
|
||||
|
||||
let pg_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM storage.blobs")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
|
||||
let sample_rows: Vec<(String, i64)> =
|
||||
sqlx::query_as("SELECT hash, size FROM storage.blobs ORDER BY random() LIMIT $1")
|
||||
.bind(sample_size as i64)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::internal_error("Migration", format!("Sample query failed: {}", e))
|
||||
})?;
|
||||
|
||||
let mut missing = Vec::new();
|
||||
let mut size_mismatches = Vec::new();
|
||||
|
||||
for (hash, expected_size) in &sample_rows {
|
||||
match target.blob_exists(hash).await {
|
||||
Ok(false) => missing.push(hash.clone()),
|
||||
Err(e) => {
|
||||
tracing::warn!("blob_exists failed for {}: {}", hash, e);
|
||||
missing.push(hash.clone());
|
||||
}
|
||||
Ok(true) => {
|
||||
if let Ok(actual_size) = target.blob_size(hash).await
|
||||
&& actual_size != *expected_size as u64
|
||||
{
|
||||
size_mismatches.push(hash.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let passed = missing.is_empty() && size_mismatches.is_empty();
|
||||
Ok(MigrationVerifyResult {
|
||||
pg_blob_count: pg_count as u64,
|
||||
sample_checked: sample_rows.len() as u64,
|
||||
missing_in_target: missing,
|
||||
size_mismatches,
|
||||
passed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Post-migration verification result — same shape as the retired
|
||||
/// `migration_job::VerificationResult` (kept identical so the admin
|
||||
/// UI's `MigrationVerifyResult` decoder needs no change).
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct MigrationVerifyResult {
|
||||
pub pg_blob_count: u64,
|
||||
pub sample_checked: u64,
|
||||
pub missing_in_target: Vec<String>,
|
||||
pub size_mismatches: Vec<String>,
|
||||
pub passed: bool,
|
||||
}
|
||||
|
||||
/// Idle-state DTO — no run has been triggered yet.
|
||||
fn idle_migration_dto() -> MigrationStateDto {
|
||||
MigrationStateDto {
|
||||
|
||||
@@ -225,7 +225,8 @@ use crate::interfaces::api::handlers::file_handler::MoveFilePayload;
|
||||
handlers::admin_handler::start_migration,
|
||||
handlers::admin_handler::pause_migration,
|
||||
handlers::admin_handler::resume_migration,
|
||||
handlers::admin_handler::verify_migration,
|
||||
// handlers::admin_handler::verify_migration retired in
|
||||
// slice 7 — superseded by `blobs_consistency?storage=<name>`.
|
||||
handlers::admin_handler::generate_encryption_key,
|
||||
// JobRegistry admin surface — production, always-on,
|
||||
// audit-logged. Retired the `/internal/trigger-*` handlers in
|
||||
|
||||
Reference in New Issue
Block a user