feat: pluggable storage backends (S3, Azure, local) with admin UI
Implement 4-phase external storage backends architecture: Phase 1 - Foundation: - BlobStorageBackend trait (application/ports/blob_storage_ports.rs) - LocalBlobBackend: extracted all tokio::fs ops from DedupService - S3BlobBackend: AWS SDK with custom endpoint support (MinIO, R2, B2) - DedupService refactored to use Arc<dyn BlobStorageBackend> Phase 2 - Admin Panel: - StorageSettingsService with DB persistence + env override - Storage tab in admin panel (backend selector, S3 form, provider presets) - GET/PUT/POST endpoints for storage settings + connection test - i18n keys (en/es) and BEM CSS Phase 3 - Migration: - MigrationBlobBackend decorator (dual-read: target-first + source fallback) - Background migration job with parallel transfers + progress tracking - Migration UI (progress bar, ETA, pause/resume/verify/complete) - 6 admin API endpoints for migration lifecycle Phase 4 - Enterprise Extras: - CachedBlobBackend: LRU disk cache for remote backends - EncryptedBlobBackend: AES-256-GCM at-rest encryption - AzureBlobBackend: Azure Blob Storage support - RetryBlobBackend: exponential backoff for transient errors - Decorator composition in DI: retry → encryption → cache All 223 tests passing, clippy clean, fmt verified.
This commit is contained in:
Diocrafts
@@ -211,6 +211,127 @@ pub struct StorageConfig {
|
||||
/// Maximum upload file size in bytes (default: 10 GB).
|
||||
/// Applied as a hard limit to WebDAV PUT and streaming uploads.
|
||||
pub max_upload_size: usize,
|
||||
/// Which blob storage backend to use (`local`, `s3`, or `azure`).
|
||||
pub backend: StorageBackendType,
|
||||
/// S3-compatible backend configuration (used when `backend == S3`).
|
||||
pub s3: Option<S3StorageConfig>,
|
||||
/// Azure Blob Storage configuration (used when `backend == Azure`).
|
||||
pub azure: Option<AzureStorageConfig>,
|
||||
/// Local disk cache for remote backends.
|
||||
pub cache: BlobCacheConfig,
|
||||
/// Client-side encryption.
|
||||
pub encryption: EncryptionConfig,
|
||||
/// Retry policy for remote backends.
|
||||
pub retry: RetryConfig,
|
||||
}
|
||||
|
||||
/// Which blob storage backend to use.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub enum StorageBackendType {
|
||||
/// Local filesystem (default).
|
||||
#[default]
|
||||
Local,
|
||||
/// Any S3-compatible object store (AWS, Backblaze B2, R2, MinIO, …).
|
||||
S3,
|
||||
/// Azure Blob Storage.
|
||||
Azure,
|
||||
}
|
||||
|
||||
/// Configuration for an S3-compatible blob storage backend.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct S3StorageConfig {
|
||||
/// Custom endpoint URL (required for non-AWS providers).
|
||||
pub endpoint_url: Option<String>,
|
||||
/// S3 bucket name.
|
||||
pub bucket: String,
|
||||
/// AWS region (default: `us-east-1`).
|
||||
pub region: String,
|
||||
/// Access key ID.
|
||||
pub access_key: String,
|
||||
/// Secret access key.
|
||||
pub secret_key: String,
|
||||
/// Force path-style access (required for MinIO, R2, some providers).
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
/// Configuration for Azure Blob Storage.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AzureStorageConfig {
|
||||
/// Azure storage account name.
|
||||
pub account_name: String,
|
||||
/// Azure storage account key.
|
||||
pub account_key: String,
|
||||
/// Container name.
|
||||
pub container: String,
|
||||
/// Optional SAS token (alternative to account key).
|
||||
pub sas_token: Option<String>,
|
||||
}
|
||||
|
||||
/// LRU local disk cache configuration for remote blob backends.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlobCacheConfig {
|
||||
/// Enable the LRU disk cache (only useful for remote backends).
|
||||
pub enabled: bool,
|
||||
/// Maximum cache size in bytes (default: 50 GB).
|
||||
pub max_size_bytes: u64,
|
||||
/// Cache directory path (default: `{root_dir}/.blob-cache`).
|
||||
pub cache_path: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for BlobCacheConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
max_size_bytes: 50 * 1024 * 1024 * 1024, // 50 GB
|
||||
cache_path: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Client-side encryption configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EncryptionConfig {
|
||||
/// Enable AES-256-GCM encryption for blobs at rest.
|
||||
pub enabled: bool,
|
||||
/// Base64-encoded 32-byte encryption key.
|
||||
pub key_base64: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for EncryptionConfig {
|
||||
#[allow(clippy::derivable_impls)]
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
key_base64: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Retry policy configuration for remote backends.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RetryConfig {
|
||||
/// Enable retry with exponential backoff.
|
||||
pub enabled: bool,
|
||||
/// Maximum number of retry attempts.
|
||||
pub max_retries: u32,
|
||||
/// Initial backoff in milliseconds.
|
||||
pub initial_backoff_ms: u64,
|
||||
/// Maximum backoff in milliseconds.
|
||||
pub max_backoff_ms: u64,
|
||||
/// Backoff multiplier.
|
||||
pub backoff_multiplier: f64,
|
||||
}
|
||||
|
||||
impl Default for RetryConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
max_retries: 3,
|
||||
initial_backoff_ms: 100,
|
||||
max_backoff_ms: 10_000,
|
||||
backoff_multiplier: 2.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for StorageConfig {
|
||||
@@ -227,6 +348,12 @@ impl Default for StorageConfig {
|
||||
parallel_threshold: 100 * 1024 * 1024, // 100 MB
|
||||
trash_retention_days: 30, // 30 days
|
||||
max_upload_size: MAX_UPLOAD_SIZE,
|
||||
backend: StorageBackendType::Local,
|
||||
s3: None,
|
||||
azure: None,
|
||||
cache: BlobCacheConfig::default(),
|
||||
encryption: EncryptionConfig::default(),
|
||||
retry: RetryConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -828,6 +955,95 @@ impl AppConfig {
|
||||
config.storage.max_upload_size = val;
|
||||
}
|
||||
|
||||
// Storage backend selection
|
||||
if let Ok(backend) = env::var("OXICLOUD_STORAGE_BACKEND") {
|
||||
match backend.to_lowercase().as_str() {
|
||||
"s3" => config.storage.backend = StorageBackendType::S3,
|
||||
"azure" => config.storage.backend = StorageBackendType::Azure,
|
||||
_ => config.storage.backend = StorageBackendType::Local,
|
||||
}
|
||||
}
|
||||
|
||||
// S3-compatible storage configuration
|
||||
if config.storage.backend == StorageBackendType::S3 {
|
||||
let bucket = env::var("OXICLOUD_S3_BUCKET").unwrap_or_default();
|
||||
if bucket.is_empty() {
|
||||
tracing::warn!("OXICLOUD_STORAGE_BACKEND=s3 but OXICLOUD_S3_BUCKET is not set");
|
||||
}
|
||||
config.storage.s3 = Some(S3StorageConfig {
|
||||
endpoint_url: env::var("OXICLOUD_S3_ENDPOINT_URL").ok(),
|
||||
bucket,
|
||||
region: env::var("OXICLOUD_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()),
|
||||
access_key: env::var("OXICLOUD_S3_ACCESS_KEY").unwrap_or_default(),
|
||||
secret_key: env::var("OXICLOUD_S3_SECRET_KEY").unwrap_or_default(),
|
||||
force_path_style: env::var("OXICLOUD_S3_FORCE_PATH_STYLE")
|
||||
.map(|v| v.parse::<bool>().unwrap_or(false))
|
||||
.unwrap_or(false),
|
||||
});
|
||||
}
|
||||
|
||||
// Azure Blob Storage configuration
|
||||
if config.storage.backend == StorageBackendType::Azure {
|
||||
let container = env::var("OXICLOUD_AZURE_CONTAINER").unwrap_or_default();
|
||||
if container.is_empty() {
|
||||
tracing::warn!(
|
||||
"OXICLOUD_STORAGE_BACKEND=azure but OXICLOUD_AZURE_CONTAINER is not set"
|
||||
);
|
||||
}
|
||||
config.storage.azure = Some(AzureStorageConfig {
|
||||
account_name: env::var("OXICLOUD_AZURE_ACCOUNT_NAME").unwrap_or_default(),
|
||||
account_key: env::var("OXICLOUD_AZURE_ACCOUNT_KEY").unwrap_or_default(),
|
||||
container,
|
||||
sas_token: env::var("OXICLOUD_AZURE_SAS_TOKEN").ok(),
|
||||
});
|
||||
}
|
||||
|
||||
// Blob cache configuration
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_CACHE_ENABLED") {
|
||||
config.storage.cache.enabled = v.parse::<bool>().unwrap_or(false);
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_CACHE_MAX_SIZE")
|
||||
&& let Ok(bytes) = v.parse::<u64>()
|
||||
{
|
||||
config.storage.cache.max_size_bytes = bytes;
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_CACHE_PATH") {
|
||||
config.storage.cache.cache_path = Some(v);
|
||||
}
|
||||
|
||||
// Encryption configuration
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_ENCRYPTION_ENABLED") {
|
||||
config.storage.encryption.enabled = v.parse::<bool>().unwrap_or(false);
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_ENCRYPTION_KEY") {
|
||||
config.storage.encryption.key_base64 = Some(v);
|
||||
}
|
||||
|
||||
// Retry configuration
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_ENABLED") {
|
||||
config.storage.retry.enabled = v.parse::<bool>().unwrap_or(true);
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_MAX_RETRIES")
|
||||
&& let Ok(n) = v.parse::<u32>()
|
||||
{
|
||||
config.storage.retry.max_retries = n;
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_INITIAL_BACKOFF_MS")
|
||||
&& let Ok(n) = v.parse::<u64>()
|
||||
{
|
||||
config.storage.retry.initial_backoff_ms = n;
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_MAX_BACKOFF_MS")
|
||||
&& let Ok(n) = v.parse::<u64>()
|
||||
{
|
||||
config.storage.retry.max_backoff_ms = n;
|
||||
}
|
||||
if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_BACKOFF_MULTIPLIER")
|
||||
&& let Ok(n) = v.parse::<f64>()
|
||||
{
|
||||
config.storage.retry.backoff_multiplier = n;
|
||||
}
|
||||
|
||||
// OIDC configuration
|
||||
if let Ok(v) = env::var("OXICLOUD_OIDC_ENABLED") {
|
||||
config.oidc.enabled = v.parse::<bool>().unwrap_or(false);
|
||||
|
||||
+118
-1
@@ -2,10 +2,14 @@ use sqlx::PgPool;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
|
||||
use crate::common::config::StorageBackendType;
|
||||
use crate::infrastructure::db::DbPools;
|
||||
|
||||
use crate::application::services::admin_settings_service::AdminSettingsService;
|
||||
use crate::application::services::auth_application_service::AuthApplicationService;
|
||||
use crate::application::services::storage_settings_service::StorageSettingsService;
|
||||
use crate::infrastructure::services::migration_blob_backend::MigrationState;
|
||||
|
||||
use crate::application::ports::file_ports::FileUseCaseFactory;
|
||||
use crate::application::services::favorites_service::FavoritesService;
|
||||
@@ -153,10 +157,110 @@ impl AppServiceFactory {
|
||||
);
|
||||
image_transcode_service.initialize().await?;
|
||||
|
||||
// Build blob storage backend based on configuration
|
||||
let base_backend: Arc<dyn BlobStorageBackend> = match self.config.storage.backend {
|
||||
StorageBackendType::S3 => {
|
||||
let s3_config = self
|
||||
.config
|
||||
.storage
|
||||
.s3
|
||||
.as_ref()
|
||||
.expect("S3 config required when OXICLOUD_STORAGE_BACKEND=s3");
|
||||
Arc::new(
|
||||
crate::infrastructure::services::s3_blob_backend::S3BlobBackend::new(s3_config),
|
||||
)
|
||||
}
|
||||
StorageBackendType::Azure => {
|
||||
let az_config = self
|
||||
.config
|
||||
.storage
|
||||
.azure
|
||||
.as_ref()
|
||||
.expect("Azure config required when OXICLOUD_STORAGE_BACKEND=azure");
|
||||
Arc::new(
|
||||
crate::infrastructure::services::azure_blob_backend::AzureBlobBackend::new(
|
||||
az_config,
|
||||
),
|
||||
)
|
||||
}
|
||||
StorageBackendType::Local => Arc::new(
|
||||
crate::infrastructure::services::local_blob_backend::LocalBlobBackend::new(
|
||||
&self.storage_path,
|
||||
),
|
||||
),
|
||||
};
|
||||
|
||||
// Stack decorators: retry → encryption → cache (inner-to-outer)
|
||||
let mut blob_backend: Arc<dyn BlobStorageBackend> = base_backend;
|
||||
|
||||
// Retry decorator (for remote backends)
|
||||
if self.config.storage.retry.enabled
|
||||
&& self.config.storage.backend != StorageBackendType::Local
|
||||
{
|
||||
use crate::infrastructure::services::retry_blob_backend::{
|
||||
RetryBlobBackend, RetryPolicy,
|
||||
};
|
||||
let policy = RetryPolicy {
|
||||
max_retries: self.config.storage.retry.max_retries,
|
||||
initial_backoff: std::time::Duration::from_millis(
|
||||
self.config.storage.retry.initial_backoff_ms,
|
||||
),
|
||||
max_backoff: std::time::Duration::from_millis(
|
||||
self.config.storage.retry.max_backoff_ms,
|
||||
),
|
||||
backoff_multiplier: self.config.storage.retry.backoff_multiplier,
|
||||
};
|
||||
blob_backend = Arc::new(RetryBlobBackend::new(blob_backend, policy));
|
||||
tracing::info!("Blob storage retry decorator enabled");
|
||||
}
|
||||
|
||||
// Encryption decorator
|
||||
if self.config.storage.encryption.enabled {
|
||||
use crate::infrastructure::services::encrypted_blob_backend::EncryptedBlobBackend;
|
||||
let key_b64 = self
|
||||
.config
|
||||
.storage
|
||||
.encryption
|
||||
.key_base64
|
||||
.as_ref()
|
||||
.expect("OXICLOUD_STORAGE_ENCRYPTION_KEY required when encryption is enabled");
|
||||
let key_bytes =
|
||||
base64::Engine::decode(&base64::engine::general_purpose::STANDARD, key_b64)
|
||||
.expect("OXICLOUD_STORAGE_ENCRYPTION_KEY must be valid base64");
|
||||
let key: [u8; 32] = key_bytes.try_into().expect(
|
||||
"OXICLOUD_STORAGE_ENCRYPTION_KEY must be exactly 32 bytes (base64 of 32 bytes)",
|
||||
);
|
||||
blob_backend = Arc::new(EncryptedBlobBackend::new(blob_backend, &key));
|
||||
tracing::info!("Blob storage encryption decorator enabled (AES-256-GCM)");
|
||||
}
|
||||
|
||||
// Cache decorator (for remote backends only)
|
||||
if self.config.storage.cache.enabled
|
||||
&& self.config.storage.backend != StorageBackendType::Local
|
||||
{
|
||||
use crate::infrastructure::services::cached_blob_backend::{
|
||||
BlobCacheConfig as CacheCfg, CachedBlobBackend,
|
||||
};
|
||||
let cache_path = self
|
||||
.config
|
||||
.storage
|
||||
.cache
|
||||
.cache_path
|
||||
.as_ref()
|
||||
.map(std::path::PathBuf::from)
|
||||
.unwrap_or_else(|| self.storage_path.join(".blob-cache"));
|
||||
let cfg = CacheCfg {
|
||||
cache_dir: cache_path,
|
||||
max_cache_bytes: self.config.storage.cache.max_size_bytes,
|
||||
};
|
||||
blob_backend = Arc::new(CachedBlobBackend::new(blob_backend, &cfg));
|
||||
tracing::info!("Blob storage LRU disk cache enabled");
|
||||
}
|
||||
|
||||
// Deduplication service — PRIMARY blob storage engine (PostgreSQL-backed index)
|
||||
let dedup_service = Arc::new(
|
||||
crate::infrastructure::services::dedup_service::DedupService::new(
|
||||
&self.storage_path,
|
||||
blob_backend,
|
||||
db_pool.clone(),
|
||||
maintenance_pool.clone(),
|
||||
),
|
||||
@@ -631,6 +735,8 @@ impl AppServiceFactory {
|
||||
auth_service: auth_services,
|
||||
nextcloud: nextcloud_services,
|
||||
admin_settings_service: None,
|
||||
storage_settings_service: None,
|
||||
migration_state: Arc::new(tokio::sync::RwLock::new(MigrationState::default())),
|
||||
trash_service,
|
||||
share_service,
|
||||
favorites_service,
|
||||
@@ -700,6 +806,15 @@ impl AppServiceFactory {
|
||||
|
||||
app_state.admin_settings_service = Some(admin_svc.clone());
|
||||
|
||||
// 9b-1b. Wire storage settings service (reuses same settings_repo)
|
||||
let storage_settings_svc = Arc::new(StorageSettingsService::new(
|
||||
settings_repo.clone(),
|
||||
self.config.storage.clone(),
|
||||
app_state.core.dedup_service.clone(),
|
||||
));
|
||||
app_state.storage_settings_service = Some(storage_settings_svc);
|
||||
tracing::info!("Storage settings service initialized");
|
||||
|
||||
// 9b-2. Log whether system needs first-time admin setup
|
||||
if !admin_svc.is_system_initialized().await {
|
||||
tracing::warn!("╔══════════════════════════════════════════════════════════╗");
|
||||
@@ -928,6 +1043,8 @@ pub struct AppState {
|
||||
pub auth_service: Option<AuthServices>,
|
||||
pub nextcloud: Option<NextcloudServices>,
|
||||
pub admin_settings_service: Option<Arc<AdminSettingsService>>,
|
||||
pub storage_settings_service: Option<Arc<StorageSettingsService>>,
|
||||
pub migration_state: Arc<tokio::sync::RwLock<MigrationState>>,
|
||||
pub trash_service: Option<Arc<TrashService>>,
|
||||
pub share_service: Option<Arc<ShareService>>,
|
||||
pub favorites_service: Option<Arc<FavoritesService>>,
|
||||
|
||||
Reference in New Issue
Block a user