cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(storage): improve admin panel

Browse Source
This commit is contained in:
Edouard Vanbelle
2026-08-01 16:37:23 +02:00
parent cc439aaff9
commit 8329b4aa56
15 changed files with 1106 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/application/dtos/settings_dto.rs
+42 -24
View File
@@ -143,31 +143,29 @@ pub struct DashboardStatsDto {
// Storage Settings DTOs (Admin Panel)
// ============================================================================
/// Current storage settings returned to admin UI (secrets masked).
/// Current storage settings returned to admin UI.
///
/// Post-multi-entry (`docs/plan/storage-multi-entry.md`) this carries
/// two shapes side-by-side: the legacy flat storage-config fields
/// (for the pre-multi-entry admin UI, until slice 6 completes the
/// form retirement), plus the new `entries` / `active_entry_name` /
/// `migration_readonly` view the multi-entry UI drives its entries-list,
/// migration-target-dropdown, and readonly-banner from.
/// Post-multi-entry (`docs/plan/storage-multi-entry.md`) this exposes:
/// - the entries declared in `.env` (safe: `location_hint` shows
/// provider+bucket, credential-related fields never appear),
/// - the name of the active entry (backend selection is admin-observable),
/// - the read-only flag (drives the UI banner during migration),
/// - the currently-live backend type + dedup stats (informational).
///
/// The pre-multi-entry `s3_*` / `backend` / `env_overrides` fields
/// used to also appear here — they duplicated `entries[]` and leaked
/// stale legacy admin_settings rows, so slice-6 dropped them. Consumers
/// wanting per-provider details read them off `entries[i].backend` and
/// `entries[i].location_hint` instead.
#[derive(Debug, Serialize, Deserialize)]
pub struct StorageSettingsDto {
/// Active backend type: "local" or "s3". Legacy flat-config field
/// — mirrors `entries[i where is_active].backend` for the active
/// entry when multi-entry is in use.
pub backend: String,
pub s3_endpoint_url: Option<String>,
pub s3_bucket: Option<String>,
pub s3_region: Option<String>,
/// True if an access key is configured (never reveals the actual value)
pub s3_access_key_set: bool,
/// True if a secret key is configured (never reveals the actual value)
pub s3_secret_key_set: bool,
pub s3_force_path_style: bool,
/// Field names overridden by environment variables (read-only in UI)
pub env_overrides: Vec<String>,
// ── Current stats ──
// ── Current stats — pertain to the running process ──
/// Backend type currently in use (`"local"` / `"s3"` / `"azure"`) —
/// what the LIVE `blob_backend` is bound to, from
/// `dedup_service.backend().backend_type()`. Redundant with
/// `entries[i where is_active].backend` in multi-entry mode; kept
/// because pre-boot / mid-migration inspection may still find it
/// useful.
pub current_backend: String,
pub total_blobs: u64,
pub total_bytes_stored: u64,
@@ -229,9 +227,25 @@ pub struct SaveStorageSettingsDto {
pub s3_force_path_style: Option<bool>,
}
/// Request body for testing a storage connection
#[derive(Debug, Serialize, Deserialize)]
/// Request body for testing a storage connection.
///
/// Two shapes are accepted:
///
/// - Multi-entry test — set `entry_name` to the name of a declared entry
/// (from `OXICLOUD_STORAGE_ENTRIES`). Server looks it up, builds a fresh
/// backend via the shared factory, runs health-check + round-trip against
/// it. `backend` and the S3 fields are ignored in this mode.
/// - Legacy DTO test — leave `entry_name` unset and populate `backend` +
/// the S3 fields. Server builds a temporary backend from those values
/// (pre-multi-entry behaviour). Still supported for zero-entries
/// deployments; deprecated for new integrations.
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct TestStorageConnectionDto {
/// If set, all other fields are ignored — server resolves this
/// name against `OXICLOUD_STORAGE_ENTRIES` and tests that entry.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub entry_name: Option<String>,
#[serde(default)]
pub backend: String,
pub s3_endpoint_url: Option<String>,
pub s3_bucket: Option<String>,
@@ -241,6 +255,10 @@ pub struct TestStorageConnectionDto {
pub s3_force_path_style: Option<bool>,
}
// Default is derived — every field is either an Option (defaults to
// None) or `backend: String` (empty via `#[serde(default)]`). The
// hand-rolled impl was flagged by clippy::derivable_impls.
/// Result of a storage connection + round-trip test.
///
/// `connected` is TRUE when the backend was reachable (health-check
src/application/services/storage_settings_service.rs
+116 -56
View File
@@ -63,26 +63,6 @@ impl StorageSettingsService {
}
}
/// Detect which storage fields are overridden by environment variables.
fn get_env_overrides(&self) -> Vec<String> {
let mut out = Vec::new();
let vars = [
("OXICLOUD_STORAGE_BACKEND", "backend"),
("OXICLOUD_S3_ENDPOINT_URL", "s3_endpoint_url"),
("OXICLOUD_S3_BUCKET", "s3_bucket"),
("OXICLOUD_S3_REGION", "s3_region"),
("OXICLOUD_S3_ACCESS_KEY", "s3_access_key"),
("OXICLOUD_S3_SECRET_KEY", "s3_secret_key"),
("OXICLOUD_S3_FORCE_PATH_STYLE", "s3_force_path_style"),
];
for (env_key, field_name) in &vars {
if std::env::var(env_key).is_ok() {
out.push(field_name.to_string());
}
}
out
}
/// Apply environment variable overrides on top of a config.
fn apply_env_overrides(&self, config: &mut StorageConfig) {
let e = &self.env_storage_config;
@@ -258,36 +238,16 @@ impl StorageSettingsService {
Ok(config)
}
/// Get storage settings for display in admin UI (secrets masked).
/// Get storage settings for display in admin UI.
///
/// Post-slice-6 the response is the multi-entry view + live stats
/// only. The legacy `backend` / `s3_*` / `env_overrides` fields
/// were retired — they duplicated `entries[]` and leaked stale
/// admin_settings rows saved by the retired admin-panel form.
pub async fn get_storage_settings(&self) -> Result<StorageSettingsDto, DomainError> {
let db: HashMap<String, String> = self.settings_repo.get_by_category("storage").await?;
let has_access_key = db
.get("storage.s3.access_key")
.map(|s| !s.is_empty())
.unwrap_or(false)
|| std::env::var("OXICLOUD_S3_ACCESS_KEY")
.map(|s| !s.is_empty())
.unwrap_or(false);
let has_secret_key = db
.get("storage.s3.secret_key")
.map(|s| !s.is_empty())
.unwrap_or(false)
|| std::env::var("OXICLOUD_S3_SECRET_KEY")
.map(|s| !s.is_empty())
.unwrap_or(false);
let effective = self.load_effective_storage_config().await?;
let stats = self.dedup_service.get_stats().await;
let current_backend = self.dedup_service.backend().backend_type().to_string();
let backend_str = match effective.backend {
crate::common::config::StorageBackendType::Local => "local",
crate::common::config::StorageBackendType::S3 => "s3",
crate::common::config::StorageBackendType::Azure => "azure",
};
// Project the multi-entry view. `is_active` is name-compared
// against the boot-selected `active_entry_name` (matches
// exactly one entry when we're in multi-entry mode; matches
@@ -310,14 +270,6 @@ impl StorageSettingsService {
.collect();
Ok(StorageSettingsDto {
backend: backend_str.to_string(),
s3_endpoint_url: effective.s3.as_ref().and_then(|s| s.endpoint_url.clone()),
s3_bucket: effective.s3.as_ref().map(|s| s.bucket.clone()),
s3_region: effective.s3.as_ref().map(|s| s.region.clone()),
s3_access_key_set: has_access_key,
s3_secret_key_set: has_secret_key,
s3_force_path_style: effective.s3.as_ref().is_some_and(|s| s.force_path_style),
env_overrides: self.get_env_overrides(),
current_backend,
total_blobs: stats.total_blobs,
total_bytes_stored: stats.total_bytes_stored,
@@ -399,6 +351,86 @@ impl StorageSettingsService {
&self,
dto: TestStorageConnectionDto,
) -> Result<StorageTestResultDto, DomainError> {
// Multi-entry mode: `entry_name` present → look up the entry,
// build via the shared factory, health-check + round-trip. The
// legacy DTO fields (backend / s3_*) are ignored. Unknown
// names return a `connected: false` result with a clear
// message, mirroring the shape of the legacy path (no
// exceptions for the client to catch — errors are inline).
if let Some(name) = dto.entry_name.as_deref() {
let entry = match self.storage_entries.iter().find(|e| e.name == name) {
Some(e) => e,
None => {
let available = if self.storage_entries.is_empty() {
"(none)".to_string()
} else {
self.storage_entries
.iter()
.map(|e| e.name.as_str())
.collect::<Vec<_>>()
.join(", ")
};
return Ok(StorageTestResultDto {
connected: false,
message: format!(
"entry `{name}` not declared in OXICLOUD_STORAGE_ENTRIES. \
Available: [{available}]"
),
backend_type: "unknown".to_string(),
available_bytes: None,
roundtrip_passed: None,
phase_reached: None,
bytes_written: None,
bytes_read: None,
roundtrip_elapsed_ms: None,
cleanup_ok: None,
});
}
};
let backend = crate::infrastructure::services::entry_backend::build_entry_backend(
entry,
std::path::Path::new(&self.env_storage_config.root_dir),
);
let backend_kind = backend.backend_type().to_string();
let status = match backend.health_check().await {
Ok(s) => s,
Err(e) => {
return Ok(StorageTestResultDto {
connected: false,
message: format!("health-check failed: {e}"),
backend_type: backend_kind,
available_bytes: None,
roundtrip_passed: None,
phase_reached: None,
bytes_written: None,
bytes_read: None,
roundtrip_elapsed_ms: None,
cleanup_ok: None,
});
}
};
let mut out = StorageTestResultDto {
connected: status.connected,
message: status.message,
backend_type: backend_kind,
available_bytes: status.available_bytes,
roundtrip_passed: None,
phase_reached: None,
bytes_written: None,
bytes_read: None,
roundtrip_elapsed_ms: None,
cleanup_ok: None,
};
if out.connected {
attach_roundtrip(&mut out, backend.as_ref()).await;
}
return Ok(out);
}
// Legacy path — DTO carries backend + s3 fields directly.
// Retained for pre-multi-entry deployments (zero declared
// entries) and for the admin form's on-form-values Test
// button. Deprecated for new integrations.
match dto.backend.as_str() {
"local" => {
// Local: no per-DTO override for the root_dir (the
@@ -727,7 +759,35 @@ async fn run_backend_roundtrip(
fn entry_location_hint(entry: &NamedStorageEntry) -> Option<String> {
match entry.backend {
StorageBackendType::Local => entry.root_dir.clone(),
StorageBackendType::S3 => entry.s3.as_ref().map(|s3| s3.bucket.clone()),
StorageBackendType::Azure => entry.azure.as_ref().map(|az| az.container.clone()),
// S3: show `<endpoint>/<bucket>` — bucket alone can collide
// across providers (a "my-bucket" on AWS vs the same name on
// MinIO / R2 look identical without the endpoint). `aws` is
// the visual stand-in when no custom endpoint is configured
// (i.e. talking to real AWS S3).
StorageBackendType::S3 => entry.s3.as_ref().map(|s3| {
// Trim trailing `/` so an env value of `https://host/`
// doesn't render as `https://host//bucket`. Both shapes
// are legitimate env inputs (some providers publish the
// trailing slash in their docs).
let endpoint = s3
.endpoint_url
.as_deref()
.unwrap_or("aws")
.trim_end_matches('/');
format!("{endpoint}/{}", s3.bucket)
}),
// Azure: show `<endpoint_or_account>/<container>`. The
// endpoint-URL override is uncommon on Azure (Azurite
// emulator, private stamps), so fall back to the account
// name when unset — matches what a reader would look for in
// the portal. Same trailing-slash trim as S3.
StorageBackendType::Azure => entry.azure.as_ref().map(|az| {
let host = az
.endpoint_url
.as_deref()
.unwrap_or(az.account_name.as_str())
.trim_end_matches('/');
format!("{host}/{}", az.container)
}),
}
}
src/common/config.rs
+110
View File
@@ -383,6 +383,45 @@ impl Default for RetryConfig {
}
}
/// AES-GCM / AEAD cipher choice for a `NamedStorageEntry`.
///
/// Today the only shipping variant is `Aes256Gcm` — the same cipher
/// `EncryptedBlobBackend` has always hardcoded. The enum is
/// future-proofing so `OXICLOUD_STORAGE_<N>_ENCRYPTION_CIPHER` is
/// already an accepted knob when a second cipher lands
/// (chacha20-poly1305, aes-256-siv, …). Absent + `_ENCRYPTION_KEY`
/// present → defaults to `Aes256Gcm` for back-compat with entries
/// declared before this field existed.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncryptionCipher {
/// AES-256 in Galois/Counter Mode. 96-bit nonce + 128-bit tag,
/// random nonce per blob. Layout on disk / S3:
/// `[12-byte nonce] [ciphertext] [16-byte GCM tag]`.
Aes256Gcm,
}
impl EncryptionCipher {
/// Stable env-var value → variant. Case-insensitive. Extend with
/// new variants as new ciphers land; the surface is one match
/// arm here + one branch in `EncryptedBlobBackend::new_for_cipher`
/// (when that gets added).
pub fn parse(raw: &str) -> Option<Self> {
match raw.to_ascii_lowercase().as_str() {
"aes-256-gcm" | "aes256gcm" => Some(EncryptionCipher::Aes256Gcm),
_ => None,
}
}
/// Stable env-var-friendly name — the exact string
/// `OXICLOUD_STORAGE_<N>_ENCRYPTION_CIPHER` accepts, and what
/// admin surfaces render back to operators.
pub fn as_str(self) -> &'static str {
match self {
EncryptionCipher::Aes256Gcm => "aes-256-gcm",
}
}
}
/// One named storage entry declared in `.env`.
///
/// See `docs/plan/storage-multi-entry.md`. Each entry is a fully-realised
@@ -417,6 +456,17 @@ pub struct NamedStorageEntry {
/// separate `_ENCRYPTION_ENABLED` toggle. Absence = raw backend.
/// Validated at parse time; boot aborts on invalid key.
pub encryption_key_base64: Option<String>,
/// Cipher choice for this entry. `Some(Aes256Gcm)` today when
/// `_ENCRYPTION_KEY` is set (whether or not the operator
/// declared `_ENCRYPTION_CIPHER=aes-256-gcm` explicitly, since
/// that's currently the only supported variant).
/// `None` when no encryption key is set. When new ciphers land,
/// the parser reads `_ENCRYPTION_CIPHER` to distinguish; today
/// the field is effectively a boolean because there's exactly
/// one variant. Kept as an enum on the type so downstream code
/// pattern-matches the concept explicitly and we don't lose the
/// intent when a second cipher is added.
pub encryption_cipher: Option<EncryptionCipher>,
}
/// Validation for a `NamedStorageEntry.name`. Restricts to a safe subset
@@ -499,6 +549,25 @@ pub fn parse_storage_entries() -> Result<Vec<NamedStorageEntry>, String> {
// for `storage.backend` / `storage.s3` / `storage.azure` /
// `storage.encryption`; centralised here so the new entry
// model and the legacy path stay bit-identical.
//
// Emit a deprecation warning naming every legacy var we saw
// so operators see the exact cleanup list in boot logs. Also
// logs to `target: "audit"` so it lands on the operational
// stream that gets watched by monitoring, not just the debug
// channel. Removal target isn't fixed yet — legacy still
// works — but the earlier operators start migrating, the
// less churn when we do pull the plug.
let names: Vec<&str> = legacy_present.iter().map(|v| **v).collect();
tracing::warn!(
target: "audit",
event = "storage.legacy_flat_vars_deprecated",
vars = ?names,
"⚠️ DEPRECATED: booted with legacy flat storage vars ({vars:?}). \
Migrate each var into `OXICLOUD_STORAGE_<NAME>_*` under an entry \
declared in `OXICLOUD_STORAGE_ENTRIES`. See \
docs/plan/storage-multi-entry.md §'Legacy flat-var interaction'.",
vars = names,
);
return Ok(vec![synthesize_default_from_legacy_vars()?]);
}
@@ -657,6 +726,38 @@ fn parse_named_entry(name: &str) -> Result<NamedStorageEntry, String> {
_ => None,
};
// Cipher — future-proofing knob. Only `aes-256-gcm` today.
// Explicit value → parse (unknown = fail-fast so a typo isn't
// silently defaulted). Absent + key set → default to
// `Aes256Gcm`. Absent + no key → `None` (no encryption at all).
let encryption_cipher = match env::var(format!("OXICLOUD_STORAGE_{name}_ENCRYPTION_CIPHER")) {
Ok(raw) if !raw.is_empty() => match EncryptionCipher::parse(&raw) {
Some(c) => Some(c),
None => {
return Err(format!(
"OXICLOUD_STORAGE_{name}_ENCRYPTION_CIPHER=`{raw}` is not a known cipher — \
supported: `aes-256-gcm`."
));
}
},
_ => {
if encryption_key_base64.is_some() {
Some(EncryptionCipher::Aes256Gcm)
} else {
None
}
}
};
// Nonsense combo — cipher declared without a key. Refuse rather
// than silently ignore the operator's declared intent.
if encryption_cipher.is_some() && encryption_key_base64.is_none() {
return Err(format!(
"OXICLOUD_STORAGE_{name}_ENCRYPTION_CIPHER is set but \
OXICLOUD_STORAGE_{name}_ENCRYPTION_KEY is not — set the key too, or remove the cipher."
));
}
Ok(NamedStorageEntry {
name: name.to_string(),
backend,
@@ -664,6 +765,7 @@ fn parse_named_entry(name: &str) -> Result<NamedStorageEntry, String> {
s3,
azure,
encryption_key_base64,
encryption_cipher,
})
}
@@ -736,6 +838,13 @@ fn synthesize_default_from_legacy_vars() -> Result<NamedStorageEntry, String> {
}
_ => None,
};
// Legacy synthesis: no `OXICLOUD_STORAGE_ENCRYPTION_CIPHER` env
// var exists in the legacy flat surface — always default to
// AES-256-GCM when a legacy key is set (matches the hardcoded
// pre-multi-entry behaviour).
let encryption_cipher = encryption_key_base64
.as_ref()
.map(|_| EncryptionCipher::Aes256Gcm);
Ok(NamedStorageEntry {
name: "default".to_string(),
@@ -744,6 +853,7 @@ fn synthesize_default_from_legacy_vars() -> Result<NamedStorageEntry, String> {
s3,
azure,
encryption_key_base64,
encryption_cipher,
})
}
src/infrastructure/services/dedup_service.rs
+5 -6
View File
@@ -2155,12 +2155,11 @@ impl DedupService {
// is NULL, COALESCE inlines the literal integer `0`, the row
// decodes fine, and the bug never surfaces during dev — hence
// it lasted so long.
let (total_blobs, total_bytes_stored): (i64, i64) = sqlx::query_as(
"SELECT COUNT(*), COALESCE(SUM(size), 0)::bigint FROM storage.blobs",
)
.fetch_one(self.pool.as_ref())
.await
.unwrap_or((0, 0));
let (total_blobs, total_bytes_stored): (i64, i64) =
sqlx::query_as("SELECT COUNT(*), COALESCE(SUM(size), 0)::bigint FROM storage.blobs")
.fetch_one(self.pool.as_ref())
.await
.unwrap_or((0, 0));
// Referenced bytes from CDC manifests. Same `numeric`-vs-`bigint`
// gotcha: `SUM(numeric)` → `numeric`; wrap the whole sum in
src/infrastructure/services/s3_blob_backend.rs
+92 -6
View File
@@ -399,12 +399,35 @@ impl BlobStorageBackend for S3BlobBackend {
message: format!("S3 bucket '{}' is accessible", self.bucket),
available_bytes: None,
}),
Err(e) => Ok(StorageHealthStatus {
connected: false,
backend_type: "s3".to_string(),
message: format!("S3 bucket '{}' is not accessible: {}", self.bucket, e),
available_bytes: None,
}),
Err(e) => {
// The AWS SDK's `Display` impl on `SdkError` says
// just "service error" for anything the service
// returned. The real cause — signature mismatch,
// 301 redirect (wrong region), 403 (missing IAM),
// hostname unresolvable — lives on the wrapped
// `ServiceError` / `DispatchFailure` / raw response.
// Peel it apart so the admin UI + audit stream see
// the actionable message, not the tautology.
let detail = format_s3_error(&e);
tracing::warn!(
target: "audit",
event = "storage.s3.health_check_failed",
bucket = %self.bucket,
error = %detail,
error_debug = ?e,
"S3 health check on `{}` failed",
self.bucket,
);
Ok(StorageHealthStatus {
connected: false,
backend_type: "s3".to_string(),
message: format!(
"S3 bucket '{}' is not accessible: {detail}",
self.bucket
),
available_bytes: None,
})
}
}
})
}
@@ -515,3 +538,66 @@ impl BlobStorageBackend for S3BlobBackend {
})
}
}
/// Extract an actionable error string from an aws-sdk-s3 error.
///
/// `SdkError::Display` renders literally `"service error"` when the
/// service returned a structured error, which is worse than useless
/// in the admin UI. This helper walks the error chain and produces:
///
/// - `NotFound` — bucket doesn't exist (or the credentials can't see it).
/// - `<code>: <message>` — the S3-specific error code + message the
/// service returned (e.g. `InvalidAccessKeyId: The AWS Access Key
/// Id you provided does not exist`, `SignatureDoesNotMatch: The
/// request signature we calculated does not match the signature`,
/// `PermanentRedirect: The bucket you are attempting to access must
/// be addressed using the specified endpoint`).
/// - `network error: <cause>` — DNS / TLS / TCP dispatch failure.
/// - `timeout` — request timed out.
/// - `unknown SDK error: <debug>` — anything else, with the full
/// `Debug` output so the operator + audit stream see the real cause
/// instead of `"service error"`.
fn format_s3_error<E>(err: &aws_sdk_s3::error::SdkError<E>) -> String
where
E: aws_sdk_s3::error::ProvideErrorMetadata + std::fmt::Debug,
{
use aws_sdk_s3::error::SdkError;
match err {
SdkError::ServiceError(svc) => {
let inner = svc.err();
let meta = inner.meta();
let code = meta.code().unwrap_or("<no-code>");
let msg = meta.message().unwrap_or("");
// A 404 for HeadBucket surfaces as `NotFound` on the
// typed error — normalise the string so callers filter
// on it easily.
if code.eq_ignore_ascii_case("NotFound") || code == "404" {
return "NotFound (bucket doesn't exist or no permission to see it)".to_string();
}
if msg.is_empty() {
format!("{code} (HTTP {})", svc.raw().status().as_u16())
} else {
format!("{code}: {msg}")
}
}
SdkError::DispatchFailure(d) => {
// DNS / TLS / connection refused / TCP reset land here.
if d.is_io() {
format!("network I/O error: {:?}", d.as_connector_error())
} else if d.is_timeout() {
"network timeout during dispatch".to_string()
} else if d.is_user() {
format!("client-side dispatch failure: {d:?}")
} else {
format!("dispatch failure: {d:?}")
}
}
SdkError::TimeoutError(_) => "timeout".to_string(),
SdkError::ResponseError(r) => {
format!("malformed response (HTTP {}): {r:?}", r.raw().status().as_u16())
}
SdkError::ConstructionFailure(c) => format!("request construction failed: {c:?}"),
_ => format!("unknown SDK error: {err:?}"),
}
}