Files
Oxicloud/src/application/services/storage_settings_service.rs
T
Diocrafts cd3733b459 feat: pluggable storage backends (S3, Azure, local) with admin UI
Implement 4-phase external storage backends architecture:

Phase 1 - Foundation:
- BlobStorageBackend trait (application/ports/blob_storage_ports.rs)
- LocalBlobBackend: extracted all tokio::fs ops from DedupService
- S3BlobBackend: AWS SDK with custom endpoint support (MinIO, R2, B2)
- DedupService refactored to use Arc<dyn BlobStorageBackend>

Phase 2 - Admin Panel:
- StorageSettingsService with DB persistence + env override
- Storage tab in admin panel (backend selector, S3 form, provider presets)
- GET/PUT/POST endpoints for storage settings + connection test
- i18n keys (en/es) and BEM CSS

Phase 3 - Migration:
- MigrationBlobBackend decorator (dual-read: target-first + source fallback)
- Background migration job with parallel transfers + progress tracking
- Migration UI (progress bar, ETA, pause/resume/verify/complete)
- 6 admin API endpoints for migration lifecycle

Phase 4 - Enterprise Extras:
- CachedBlobBackend: LRU disk cache for remote backends
- EncryptedBlobBackend: AES-256-GCM at-rest encryption
- AzureBlobBackend: Azure Blob Storage support
- RetryBlobBackend: exponential backoff for transient errors
- Decorator composition in DI: retry → encryption → cache

All 223 tests passing, clippy clean, fmt verified.
2026-04-14 21:33:38 +02:00

338 lines
13 KiB
Rust

use std::collections::HashMap;
use std::sync::Arc;
use uuid::Uuid;
use crate::application::dtos::settings_dto::{
SaveStorageSettingsDto, StorageSettingsDto, StorageTestResultDto, TestStorageConnectionDto,
};
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
use crate::common::config::{S3StorageConfig, StorageConfig};
use crate::common::errors::{DomainError, ErrorKind};
use crate::domain::repositories::settings_repository::SettingsRepository;
use crate::infrastructure::repositories::pg::SettingsPgRepository;
use crate::infrastructure::services::dedup_service::DedupService;
use crate::infrastructure::services::s3_blob_backend::S3BlobBackend;
/// Storage settings service — manages storage backend configuration via the admin panel.
///
/// Configuration priority: **env vars > DB settings > defaults**.
pub struct StorageSettingsService {
settings_repo: Arc<SettingsPgRepository>,
env_storage_config: StorageConfig,
dedup_service: Arc<DedupService>,
}
impl StorageSettingsService {
pub fn new(
settings_repo: Arc<SettingsPgRepository>,
env_storage_config: StorageConfig,
dedup_service: Arc<DedupService>,
) -> Self {
Self {
settings_repo,
env_storage_config,
dedup_service,
}
}
/// Detect which storage fields are overridden by environment variables.
fn get_env_overrides(&self) -> Vec<String> {
let mut out = Vec::new();
let vars = [
("OXICLOUD_STORAGE_BACKEND", "backend"),
("OXICLOUD_S3_ENDPOINT_URL", "s3_endpoint_url"),
("OXICLOUD_S3_BUCKET", "s3_bucket"),
("OXICLOUD_S3_REGION", "s3_region"),
("OXICLOUD_S3_ACCESS_KEY", "s3_access_key"),
("OXICLOUD_S3_SECRET_KEY", "s3_secret_key"),
("OXICLOUD_S3_FORCE_PATH_STYLE", "s3_force_path_style"),
];
for (env_key, field_name) in &vars {
if std::env::var(env_key).is_ok() {
out.push(field_name.to_string());
}
}
out
}
/// Apply environment variable overrides on top of a config.
fn apply_env_overrides(&self, config: &mut StorageConfig) {
let e = &self.env_storage_config;
if std::env::var("OXICLOUD_STORAGE_BACKEND").is_ok() {
config.backend = e.backend.clone();
}
// S3 env overrides — only apply if S3 config exists in env
if let Some(env_s3) = &e.s3 {
let s3 = config.s3.get_or_insert_with(|| S3StorageConfig {
endpoint_url: None,
bucket: String::new(),
region: "us-east-1".to_string(),
access_key: String::new(),
secret_key: String::new(),
force_path_style: false,
});
if std::env::var("OXICLOUD_S3_ENDPOINT_URL").is_ok() {
s3.endpoint_url = env_s3.endpoint_url.clone();
}
if std::env::var("OXICLOUD_S3_BUCKET").is_ok() {
s3.bucket = env_s3.bucket.clone();
}
if std::env::var("OXICLOUD_S3_REGION").is_ok() {
s3.region = env_s3.region.clone();
}
if std::env::var("OXICLOUD_S3_ACCESS_KEY").is_ok() {
s3.access_key = env_s3.access_key.clone();
}
if std::env::var("OXICLOUD_S3_SECRET_KEY").is_ok() {
s3.secret_key = env_s3.secret_key.clone();
}
if std::env::var("OXICLOUD_S3_FORCE_PATH_STYLE").is_ok() {
s3.force_path_style = env_s3.force_path_style;
}
}
}
/// Load effective storage config: DB settings + env var overrides + defaults.
pub async fn load_effective_storage_config(&self) -> Result<StorageConfig, DomainError> {
let db: HashMap<String, String> = self.settings_repo.get_by_category("storage").await?;
let d = StorageConfig::default();
let backend = db
.get("storage.backend")
.map(|v| match v.as_str() {
"s3" => crate::common::config::StorageBackendType::S3,
"azure" => crate::common::config::StorageBackendType::Azure,
_ => crate::common::config::StorageBackendType::Local,
})
.unwrap_or(d.backend);
let s3 = {
let bucket = db.get("storage.s3.bucket").cloned().unwrap_or_default();
if bucket.is_empty() {
None
} else {
Some(S3StorageConfig {
endpoint_url: db
.get("storage.s3.endpoint_url")
.cloned()
.filter(|s| !s.is_empty()),
bucket,
region: db
.get("storage.s3.region")
.cloned()
.unwrap_or_else(|| "us-east-1".to_string()),
access_key: db.get("storage.s3.access_key").cloned().unwrap_or_default(),
secret_key: db.get("storage.s3.secret_key").cloned().unwrap_or_default(),
force_path_style: db
.get("storage.s3.force_path_style")
.and_then(|v| v.parse().ok())
.unwrap_or(false),
})
}
};
let mut config = StorageConfig {
backend,
s3,
..self.env_storage_config.clone()
};
self.apply_env_overrides(&mut config);
Ok(config)
}
/// Get storage settings for display in admin UI (secrets masked).
pub async fn get_storage_settings(&self) -> Result<StorageSettingsDto, DomainError> {
let db: HashMap<String, String> = self.settings_repo.get_by_category("storage").await?;
let has_access_key = db
.get("storage.s3.access_key")
.map(|s| !s.is_empty())
.unwrap_or(false)
|| std::env::var("OXICLOUD_S3_ACCESS_KEY")
.map(|s| !s.is_empty())
.unwrap_or(false);
let has_secret_key = db
.get("storage.s3.secret_key")
.map(|s| !s.is_empty())
.unwrap_or(false)
|| std::env::var("OXICLOUD_S3_SECRET_KEY")
.map(|s| !s.is_empty())
.unwrap_or(false);
let effective = self.load_effective_storage_config().await?;
let stats = self.dedup_service.get_stats().await;
let current_backend = self.dedup_service.backend().backend_type().to_string();
let backend_str = match effective.backend {
crate::common::config::StorageBackendType::Local => "local",
crate::common::config::StorageBackendType::S3 => "s3",
crate::common::config::StorageBackendType::Azure => "azure",
};
Ok(StorageSettingsDto {
backend: backend_str.to_string(),
s3_endpoint_url: effective.s3.as_ref().and_then(|s| s.endpoint_url.clone()),
s3_bucket: effective.s3.as_ref().map(|s| s.bucket.clone()),
s3_region: effective.s3.as_ref().map(|s| s.region.clone()),
s3_access_key_set: has_access_key,
s3_secret_key_set: has_secret_key,
s3_force_path_style: effective.s3.as_ref().is_some_and(|s| s.force_path_style),
env_overrides: self.get_env_overrides(),
current_backend,
total_blobs: stats.total_blobs,
total_bytes_stored: stats.total_bytes_stored,
dedup_ratio: stats.dedup_ratio,
})
}
/// Save storage settings to DB.
pub async fn save_storage_settings(
&self,
dto: SaveStorageSettingsDto,
updated_by: Uuid,
) -> Result<(), DomainError> {
let cat = "storage";
let by = Some(updated_by);
self.settings_repo
.set("storage.backend", &dto.backend, cat, false, by)
.await?;
if let Some(ref v) = dto.s3_endpoint_url {
self.settings_repo
.set("storage.s3.endpoint_url", v, cat, false, by)
.await?;
}
if let Some(ref v) = dto.s3_bucket {
self.settings_repo
.set("storage.s3.bucket", v, cat, false, by)
.await?;
}
if let Some(ref v) = dto.s3_region {
self.settings_repo
.set("storage.s3.region", v, cat, false, by)
.await?;
}
if let Some(ref v) = dto.s3_access_key
&& !v.is_empty()
{
self.settings_repo
.set("storage.s3.access_key", v, cat, true, by)
.await?;
}
if let Some(ref v) = dto.s3_secret_key
&& !v.is_empty()
{
self.settings_repo
.set("storage.s3.secret_key", v, cat, true, by)
.await?;
}
if let Some(v) = dto.s3_force_path_style {
self.settings_repo
.set(
"storage.s3.force_path_style",
&v.to_string(),
cat,
false,
by,
)
.await?;
}
tracing::info!("Storage settings saved by admin (backend={})", dto.backend);
Ok(())
}
/// Test a storage connection by building a temporary backend and calling health_check().
pub async fn test_storage_connection(
&self,
dto: TestStorageConnectionDto,
) -> Result<StorageTestResultDto, DomainError> {
match dto.backend.as_str() {
"local" => {
// Test local backend health via the current dedup service backend
let status = self.dedup_service.backend().health_check().await?;
Ok(StorageTestResultDto {
connected: status.connected,
message: status.message,
backend_type: "local".to_string(),
available_bytes: status.available_bytes,
})
}
"s3" => {
let bucket = dto.s3_bucket.as_deref().unwrap_or_default();
if bucket.is_empty() {
return Ok(StorageTestResultDto {
connected: false,
message: "S3 bucket name is required".to_string(),
backend_type: "s3".to_string(),
available_bytes: None,
});
}
// Build a temporary S3 backend from the DTO values,
// falling back to existing DB/env config for missing fields.
let effective = self.load_effective_storage_config().await.ok();
let existing_s3 = effective.as_ref().and_then(|c| c.s3.as_ref());
let config = S3StorageConfig {
endpoint_url: dto
.s3_endpoint_url
.clone()
.or_else(|| existing_s3.and_then(|s| s.endpoint_url.clone())),
bucket: bucket.to_string(),
region: dto.s3_region.clone().unwrap_or_else(|| {
existing_s3
.map(|s| s.region.clone())
.unwrap_or_else(|| "us-east-1".to_string())
}),
access_key: dto
.s3_access_key
.clone()
.filter(|s| !s.is_empty())
.unwrap_or_else(|| {
existing_s3
.map(|s| s.access_key.clone())
.unwrap_or_default()
}),
secret_key: dto
.s3_secret_key
.clone()
.filter(|s| !s.is_empty())
.unwrap_or_else(|| {
existing_s3
.map(|s| s.secret_key.clone())
.unwrap_or_default()
}),
force_path_style: dto
.s3_force_path_style
.unwrap_or_else(|| existing_s3.is_some_and(|s| s.force_path_style)),
};
let backend = S3BlobBackend::new(&config);
match backend.health_check().await {
Ok(status) => Ok(StorageTestResultDto {
connected: status.connected,
message: status.message,
backend_type: "s3".to_string(),
available_bytes: status.available_bytes,
}),
Err(e) => Ok(StorageTestResultDto {
connected: false,
message: format!("Connection failed: {}", e),
backend_type: "s3".to_string(),
available_bytes: None,
}),
}
}
other => Err(DomainError::new(
ErrorKind::InvalidInput,
"Storage",
format!("Unknown backend type: {}", other),
)),
}
}
}