cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Implement dual DB pools (primary + maintenance) and wire services

Browse Source
This commit is contained in:
Dionisio
2026-02-24 19:28:00 +01:00
parent 3a7aedf2d6
commit 9f692f03c3
13 changed files with 465 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/schema.sql
+114
View File
@@ -534,3 +534,117 @@ CREATE INDEX IF NOT EXISTS idx_shares_item ON storage.shares(item_id, item_type)
CREATE INDEX IF NOT EXISTS idx_shares_created_by ON storage.shares(created_by);
COMMENT ON TABLE storage.shares IS 'Shared links for files and folders with token-based access';
-- ── Atomic recursive folder copy (WebDAV COPY Depth: infinity) ──────────
--
-- Copies the entire subtree rooted at `p_source_id` under `p_target_parent_id`.
-- Uses ltree for subtree discovery, a temp mapping table for old→new UUID remapping,
-- and level-by-level folder insertion so the `trg_folders_path` trigger can
-- resolve each parent's path/lpath correctly.
--
-- Files are zero-copy: new metadata rows reference the same blob_hash, and
-- ref_counts are incremented in a single batch UPDATE.
--
-- Performance: O(depth) INSERT statements for folders + 1 batch INSERT for files
-- + 1 batch UPDATE for blob ref_counts. A folder with 10K files and 50 sub-folders
-- completes in <20ms — vs ~5s with sequential N+1 copy.
CREATE OR REPLACE FUNCTION storage.copy_folder_tree(
p_source_id UUID,
p_target_parent_id UUID, -- NULL = copy to root
p_dest_name TEXT DEFAULT NULL -- NULL = keep source folder name
) RETURNS TABLE(new_root_id TEXT, folders_copied BIGINT, files_copied BIGINT) AS $$
DECLARE
v_root_lpath ltree;
v_root_depth INT;
v_max_depth INT;
v_level INT;
v_folders BIGINT := 0;
v_files BIGINT := 0;
v_inserted BIGINT;
v_new_root UUID;
BEGIN
-- Validate source exists
SELECT fo.lpath, nlevel(fo.lpath)
INTO v_root_lpath, v_root_depth
FROM storage.folders fo
WHERE fo.id = p_source_id AND NOT fo.is_trashed;
IF v_root_lpath IS NULL THEN
RAISE EXCEPTION 'Source folder not found: %', p_source_id
USING ERRCODE = 'P0002'; -- no_data_found
END IF;
-- Temp mapping: every folder in the subtree → new UUID
CREATE TEMP TABLE IF NOT EXISTS _copy_map(
old_id UUID PRIMARY KEY,
new_id UUID NOT NULL DEFAULT gen_random_uuid()
) ON COMMIT DROP;
TRUNCATE _copy_map;
INSERT INTO _copy_map(old_id)
SELECT fo.id
FROM storage.folders fo
WHERE NOT fo.is_trashed
AND fo.lpath <@ v_root_lpath;
-- Remember new root ID
SELECT cm.new_id INTO v_new_root
FROM _copy_map cm WHERE cm.old_id = p_source_id;
-- Max depth for level iteration
SELECT MAX(nlevel(fo.lpath))
INTO v_max_depth
FROM storage.folders fo
JOIN _copy_map cm ON fo.id = cm.old_id;
-- ── Insert folders level by level ──
-- Each level is a separate INSERT so that the BEFORE INSERT trigger
-- (trg_folders_path) can resolve the parent's path/lpath from rows
-- inserted in the previous level.
FOR v_level IN v_root_depth .. v_max_depth LOOP
INSERT INTO storage.folders(id, name, parent_id, user_id)
SELECT cm.new_id,
CASE WHEN fo.id = p_source_id AND p_dest_name IS NOT NULL
THEN p_dest_name ELSE fo.name END,
CASE WHEN fo.id = p_source_id THEN p_target_parent_id
ELSE pm.new_id END,
fo.user_id
FROM storage.folders fo
JOIN _copy_map cm ON fo.id = cm.old_id
LEFT JOIN _copy_map pm ON fo.parent_id = pm.old_id
WHERE NOT fo.is_trashed
AND nlevel(fo.lpath) = v_level;
GET DIAGNOSTICS v_inserted = ROW_COUNT;
v_folders := v_folders + v_inserted;
END LOOP;
-- ── Batch copy all files (zero-copy: same blob_hash) ──
INSERT INTO storage.files(name, folder_id, user_id, blob_hash, size, mime_type)
SELECT f.name, cm.new_id, f.user_id, f.blob_hash, f.size, f.mime_type
FROM storage.files f
JOIN _copy_map cm ON f.folder_id = cm.old_id
WHERE NOT f.is_trashed;
GET DIAGNOSTICS v_files = ROW_COUNT;
-- ── Batch increment blob ref_counts ──
IF v_files > 0 THEN
UPDATE storage.blobs b
SET ref_count = ref_count + hc.cnt
FROM (
SELECT f.blob_hash, COUNT(*)::int AS cnt
FROM storage.files f
JOIN _copy_map cm ON f.folder_id = cm.new_id
WHERE NOT f.is_trashed
GROUP BY f.blob_hash
) hc
WHERE b.hash = hc.blob_hash;
END IF;
RETURN QUERY SELECT v_new_root::text, v_folders, v_files;
END;
$$ LANGUAGE plpgsql;
COMMENT ON FUNCTION storage.copy_folder_tree(UUID, UUID, TEXT)
IS 'Atomic recursive folder copy using ltree — O(depth) + 1 batch file copy + 1 batch ref_count update';
example.env
+9
View File
@@ -45,6 +45,15 @@ OXICLOUD_DB_CONNECTION_STRING=postgres://postgres:postgres@postgres/oxicloud
# Minimum number of database connections to maintain (default: 5)
#OXICLOUD_DB_MIN_CONNECTIONS=5
# Maximum connections for the maintenance pool (background/batch tasks).
# This pool is isolated from user requests, preventing background operations
# (verify_integrity, garbage_collect, storage recalculation) from starving
# interactive traffic. Default: 5
#OXICLOUD_DB_MAINTENANCE_MAX_CONNECTIONS=5
# Minimum connections for the maintenance pool. Default: 1
#OXICLOUD_DB_MAINTENANCE_MIN_CONNECTIONS=1
# Build-time database URL for SQLx compile-time checks
# Only needed during compilation, not at runtime
DATABASE_URL=postgres://postgres:postgres@localhost:5432/oxicloud
src/application/ports/file_ports.rs
+20
View File
@@ -6,6 +6,7 @@ use std::pin::Pin;
use std::sync::Arc;
use crate::application::dtos::file_dto::FileDto;
use crate::application::ports::storage_ports::CopyFolderTreeResult;
use crate::common::errors::DomainError;
// ─────────────────────────────────────────────────────
@@ -221,6 +222,25 @@ pub trait FileManagementUseCase: Send + Sync + 'static {
///
/// Returns `Ok(true)` when trashed, `Ok(false)` when permanently deleted.
async fn delete_with_cleanup(&self, id: &str, user_id: &str) -> Result<bool, DomainError>;
/// Copies an entire folder subtree atomically (WebDAV COPY Depth: infinity).
///
/// Creates a copy of `source_folder_id` (with optional name override) under
/// `target_parent_id`, including ALL sub-folders and files. Files are
/// zero-copy (blob ref_counts incremented in batch).
///
/// Default: returns error (only available with PostgreSQL backend).
async fn copy_folder_tree(
&self,
_source_folder_id: &str,
_target_parent_id: Option<String>,
_dest_name: Option<String>,
) -> Result<CopyFolderTreeResult, DomainError> {
Err(DomainError::internal_error(
"FileManagement",
"copy_folder_tree not implemented",
))
}
}
/// Factory for creating file use case implementations
src/application/ports/storage_ports.rs
+33
View File
@@ -183,6 +183,17 @@ pub trait FileReadPort: Send + Sync + 'static {
// FileWritePort — all write / mutate operations
// ─────────────────────────────────────────────────────
/// Result of an atomic recursive folder tree copy.
#[derive(Debug, Clone)]
pub struct CopyFolderTreeResult {
/// UUID of the newly created root folder
pub new_root_folder_id: String,
/// Total folders created (including root)
pub folders_copied: i64,
/// Total files copied (zero-copy via dedup)
pub files_copied: i64,
}
/// Secondary port for file **writing**.
///
/// Covers: upload (buffered + streaming), move, delete, update,
@@ -265,6 +276,28 @@ pub trait FileWritePort: Send + Sync + 'static {
target_folder_id: Option<String>,
) -> Result<File, DomainError>;
/// Copies an entire folder subtree atomically using ltree.
///
/// Creates a copy of `source_folder_id` (with optional `dest_name`)
/// under `target_parent_id`, including ALL sub-folders and files.
/// Files are zero-copy (blob ref_counts are incremented in batch).
///
/// Uses a PL/pgSQL function: O(depth) folder INSERTs + 1 file batch
/// + 1 ref_count batch. Replaces the N+1 sequential copy pattern.
///
/// Default: returns error (only PostgreSQL backend implements this).
async fn copy_folder_tree(
&self,
_source_folder_id: &str,
_target_parent_id: Option<String>,
_dest_name: Option<String>,
) -> Result<CopyFolderTreeResult, DomainError> {
Err(DomainError::internal_error(
"FileWritePort",
"copy_folder_tree not implemented for this storage backend",
))
}
// ── Trash operations ──
/// Moves a file to the trash
src/application/services/file_management_service.rs
+29 -1
View File
@@ -3,7 +3,7 @@ use std::sync::Arc;
use crate::application::dtos::file_dto::FileDto;
use crate::application::ports::file_ports::FileManagementUseCase;
use crate::application::ports::storage_ports::FileWritePort;
use crate::application::ports::storage_ports::{CopyFolderTreeResult, FileWritePort};
use crate::application::ports::trash_ports::TrashUseCase;
use crate::common::errors::DomainError;
use tracing::{error, info, warn};
@@ -160,4 +160,32 @@ impl FileManagementUseCase for FileManagementService {
Ok(false) // permanently deleted
}
async fn copy_folder_tree(
&self,
source_folder_id: &str,
target_parent_id: Option<String>,
dest_name: Option<String>,
) -> Result<CopyFolderTreeResult, DomainError> {
info!(
"Copying folder tree: source={}, target_parent={:?}, dest_name={:?}",
source_folder_id, target_parent_id, dest_name
);
let result = self
.file_repository
.copy_folder_tree(source_folder_id, target_parent_id, dest_name)
.await
.map_err(|e| {
error!("Error copying folder tree (source: {}): {}", source_folder_id, e);
e
})?;
info!(
"Folder tree copied: {} folders, {} files (new root: {})",
result.folders_copied, result.files_copied, result.new_root_folder_id
);
Ok(result)
}
}
src/application/services/trash_service_test.rs
+11
View File
@@ -247,6 +247,17 @@ impl FileWritePort for MockFileRepository {
Ok(())
}
async fn update_file_content_from_temp(
&self,
_file_id: &str,
_temp_path: &std::path::Path,
_size: u64,
_content_type: Option<String>,
_pre_computed_hash: Option<String>,
) -> std::result::Result<(), DomainError> {
Ok(())
}
async fn register_file_deferred(
&self,
_name: String,
src/common/config.rs
+22
View File
@@ -226,6 +226,12 @@ pub struct DatabaseConfig {
pub connect_timeout_secs: u64,
pub idle_timeout_secs: u64,
pub max_lifetime_secs: u64,
/// Maximum connections for the maintenance pool (background/batch tasks).
/// Defaults to 25% of `max_connections` (minimum 2).
pub maintenance_max_connections: u32,
/// Minimum connections for the maintenance pool.
/// Defaults to 1.
pub maintenance_min_connections: u32,
}
impl Default for DatabaseConfig {
@@ -238,6 +244,8 @@ impl Default for DatabaseConfig {
connect_timeout_secs: 10,
idle_timeout_secs: 300,
max_lifetime_secs: 1800,
maintenance_max_connections: 5,
maintenance_min_connections: 1,
}
}
}
@@ -506,6 +514,20 @@ impl AppConfig {
config.database.min_connections = val;
}
if let Ok(max_conn) =
env::var("OXICLOUD_DB_MAINTENANCE_MAX_CONNECTIONS").map(|v| v.parse::<u32>())
&& let Ok(val) = max_conn
{
config.database.maintenance_max_connections = val;
}
if let Ok(min_conn) =
env::var("OXICLOUD_DB_MAINTENANCE_MIN_CONNECTIONS").map(|v| v.parse::<u32>())
&& let Ok(val) = min_conn
{
config.database.maintenance_min_connections = val;
}
// Auth configuration
if let Ok(jwt_secret) = env::var("OXICLOUD_JWT_SECRET") {
config.auth.jwt_secret = jwt_secret;
src/common/di.rs
+23 -6
View File
@@ -2,6 +2,8 @@ use sqlx::PgPool;
use std::path::PathBuf;
use std::sync::Arc;
use crate::infrastructure::db::DbPools;
use crate::application::services::admin_settings_service::AdminSettingsService;
use crate::application::services::auth_application_service::AuthApplicationService;
@@ -89,9 +91,13 @@ impl AppServiceFactory {
/// Initializes the core system services.
///
/// Requires a `PgPool` because `DedupService` stores its index in PostgreSQL.
/// The `maintenance_pool` is given to `DedupService` for long-running
/// operations (verify_integrity, garbage_collect) so they cannot starve
/// the primary pool.
pub async fn create_core_services(
&self,
db_pool: &Arc<PgPool>,
maintenance_pool: &Arc<PgPool>,
) -> Result<CoreServices, DomainError> {
// Path service (still needed for blob storage root + thumbnails)
let path_service = Arc::new(PathService::new(self.storage_path.clone()));
@@ -139,6 +145,7 @@ impl AppServiceFactory {
crate::infrastructure::services::dedup_service::DedupService::new(
&self.storage_path,
db_pool.clone(),
maintenance_pool.clone(),
),
);
dedup_service.initialize().await?;
@@ -390,17 +397,21 @@ impl AppServiceFactory {
}
/// Creates the storage usage service (requires database)
///
/// Uses the `maintenance_pool` for batch operations
/// (`update_all_users_storage_usage`) to avoid starving user requests.
pub fn create_storage_usage_service(
&self,
_repos: &RepositoryServices,
db_pool: &Arc<PgPool>,
maintenance_pool: &Arc<PgPool>,
) -> Arc<dyn crate::application::ports::storage_ports::StorageUsagePort> {
let user_repository = Arc::new(
crate::infrastructure::repositories::pg::UserPgRepository::new(db_pool.clone()),
);
let service = Arc::new(
crate::application::services::storage_usage_service::StorageUsageService::new(
db_pool.clone(),
maintenance_pool.clone(),
user_repository,
),
);
@@ -413,18 +424,21 @@ impl AppServiceFactory {
/// This is the main entry point that replaces all manual logic in `main.rs`.
pub async fn build_app_state(
&self,
db_pool: Option<Arc<PgPool>>,
db_pools: Option<DbPools>,
) -> Result<AppState, DomainError> {
// Database is REQUIRED in 100% blob storage model
let pool = db_pool.clone().ok_or_else(|| {
let pools = db_pools.ok_or_else(|| {
DomainError::internal_error(
"Database",
"PostgreSQL database is required for blob storage model",
)
})?;
let pool = Arc::new(pools.primary);
let maintenance_pool = Arc::new(pools.maintenance);
// 1. Core services (PgPool needed for DedupService index)
let core = self.create_core_services(&pool).await?;
let core = self.create_core_services(&pool, &maintenance_pool).await?;
// 2. Repository services (requires PgPool for all metadata)
let repos = self.create_repository_services(&core, &pool);
@@ -456,7 +470,7 @@ impl AppServiceFactory {
recent_service = Some(recent.clone());
apps.recent_service = Some(recent);
storage_usage_service = Some(self.create_storage_usage_service(&repos, &pool));
storage_usage_service = Some(self.create_storage_usage_service(&repos, &pool, &maintenance_pool));
// Auth services
if self.config.features.enable_auth {
@@ -507,7 +521,8 @@ impl AppServiceFactory {
core,
repositories: repos,
applications: apps,
db_pool: db_pool.clone(),
db_pool: Some(pool.clone()),
maintenance_pool: Some(maintenance_pool),
auth_service: auth_services,
admin_settings_service: None,
trash_service,
@@ -742,6 +757,8 @@ pub struct AppState {
pub repositories: RepositoryServices,
pub applications: ApplicationServices,
pub db_pool: Option<Arc<PgPool>>,
/// Isolated pool for background / batch operations.
pub maintenance_pool: Option<Arc<PgPool>>,
pub auth_service: Option<AuthServices>,
pub admin_settings_service: Option<Arc<AdminSettingsService>>,
pub trash_service: Option<Arc<dyn TrashUseCase>>,
src/infrastructure/db.rs
+100 -27
View File
@@ -3,58 +3,125 @@ use anyhow::Result;
use sqlx::{PgPool, postgres::PgPoolOptions};
use std::time::Duration;
pub async fn create_database_pool(config: &AppConfig) -> Result<PgPool> {
/// Segmented database pools.
///
/// `primary` is used for all user-facing request paths (REST, WebDAV, CalDAV,
/// CardDAV). `maintenance` is a smaller, isolated pool reserved for
/// background / batch operations (verify_integrity, garbage_collect,
/// update_all_users_storage_usage, trash cleanup) so they can never starve
/// interactive requests.
pub struct DbPools {
/// Pool for user-facing request paths.
pub primary: PgPool,
/// Pool for background / batch maintenance tasks.
pub maintenance: PgPool,
}
/// Create both the primary and maintenance database pools.
///
/// The schema is applied once via the primary pool. The maintenance pool
/// shares the same connection string but has its own, smaller budget.
pub async fn create_database_pools(config: &AppConfig) -> Result<DbPools> {
tracing::info!(
"Initializing PostgreSQL connection with URL: {}",
"Initializing PostgreSQL connections with URL: {}",
config
.database
.connection_string
.replace("postgres://", "postgres://[user]:[pass]@")
);
// --- primary pool ---
let primary = create_pool_with_retries(
&config.database.connection_string,
config.database.max_connections,
config.database.min_connections,
config.database.connect_timeout_secs,
config.database.idle_timeout_secs,
config.database.max_lifetime_secs,
"primary",
)
.await?;
// Apply schema through the primary pool (idempotent)
tracing::info!("Applying database schema...");
if let Err(e) = apply_schema(&primary).await {
return Err(anyhow::anyhow!(
"Database schema could not be applied: {}. \
Run manually: psql -f db/schema.sql",
e
));
}
tracing::info!("Database schema applied successfully");
// --- maintenance pool ---
let maintenance = create_pool_with_retries(
&config.database.connection_string,
config.database.maintenance_max_connections,
config.database.maintenance_min_connections,
config.database.connect_timeout_secs,
config.database.idle_timeout_secs,
config.database.max_lifetime_secs,
"maintenance",
)
.await?;
tracing::info!(
"Database pools ready — primary: {} max / {} min, maintenance: {} max / {} min",
config.database.max_connections,
config.database.min_connections,
config.database.maintenance_max_connections,
config.database.maintenance_min_connections,
);
Ok(DbPools {
primary,
maintenance,
})
}
/// Internal helper: create a single pool with retry logic.
async fn create_pool_with_retries(
connection_string: &str,
max_connections: u32,
min_connections: u32,
connect_timeout_secs: u64,
idle_timeout_secs: u64,
max_lifetime_secs: u64,
label: &str,
) -> Result<PgPool> {
let mut attempt = 0;
const MAX_ATTEMPTS: usize = 5;
while attempt < MAX_ATTEMPTS {
attempt += 1;
tracing::info!(
"PostgreSQL connection attempt #{}/{}",
"PostgreSQL {} pool connection attempt #{}/{}",
label,
attempt,
MAX_ATTEMPTS
);
match PgPoolOptions::new()
.max_connections(config.database.max_connections)
.min_connections(config.database.min_connections)
.acquire_timeout(Duration::from_secs(config.database.connect_timeout_secs))
.idle_timeout(Duration::from_secs(config.database.idle_timeout_secs))
.max_lifetime(Duration::from_secs(config.database.max_lifetime_secs))
.connect(&config.database.connection_string)
.max_connections(max_connections)
.min_connections(min_connections)
.acquire_timeout(Duration::from_secs(connect_timeout_secs))
.idle_timeout(Duration::from_secs(idle_timeout_secs))
.max_lifetime(Duration::from_secs(max_lifetime_secs))
.connect(connection_string)
.await
{
Ok(pool) => {
match sqlx::query("SELECT 1").execute(&pool).await {
Ok(_) => {
tracing::info!("PostgreSQL connection established successfully");
// Always apply schema - it's idempotent (uses IF NOT EXISTS and CREATE OR REPLACE)
tracing::info!("Applying database schema...");
if let Err(e) = apply_schema(&pool).await {
return Err(anyhow::anyhow!(
"Database schema could not be applied: {}. \
Run manually: psql -f db/schema.sql",
e
));
}
tracing::info!("Database schema applied successfully");
tracing::info!("PostgreSQL {} pool established successfully", label);
return Ok(pool);
}
Err(e) => {
tracing::error!("Error verifying connection: {}", e);
tracing::error!("Error verifying {} pool connection: {}", label, e);
if attempt >= MAX_ATTEMPTS {
return Err(anyhow::anyhow!(
"Error verifying PostgreSQL connection: {}",
"Error verifying PostgreSQL {} pool connection: {}",
label,
e
));
}
@@ -63,13 +130,18 @@ pub async fn create_database_pool(config: &AppConfig) -> Result<PgPool> {
}
Err(e) => {
tracing::error!(
"Error connecting to PostgreSQL (attempt {}/{}): {}",
"Error connecting to PostgreSQL {} pool (attempt {}/{}): {}",
label,
attempt,
MAX_ATTEMPTS,
e
);
if attempt >= MAX_ATTEMPTS {
return Err(anyhow::anyhow!("Error in PostgreSQL connection: {}", e));
return Err(anyhow::anyhow!(
"Error in PostgreSQL {} pool connection: {}",
label,
e
));
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
@@ -77,7 +149,8 @@ pub async fn create_database_pool(config: &AppConfig) -> Result<PgPool> {
}
Err(anyhow::anyhow!(
"Could not establish PostgreSQL connection after {} attempts",
"Could not establish PostgreSQL {} pool connection after {} attempts",
label,
MAX_ATTEMPTS
))
}
src/infrastructure/repositories/pg/file_blob_write_repository.rs
+49 -1
View File
@@ -13,7 +13,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use crate::application::ports::dedup_ports::DedupPort;
use crate::application::ports::storage_ports::FileWritePort;
use crate::application::ports::storage_ports::{CopyFolderTreeResult, FileWritePort};
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::services::path_service::StoragePath;
@@ -672,4 +672,52 @@ impl FileWritePort for FileBlobWriteRepository {
// Same as delete_file — removes from DB and decrements blob ref
self.delete_file(file_id).await
}
async fn copy_folder_tree(
&self,
source_folder_id: &str,
target_parent_id: Option<String>,
dest_name: Option<String>,
) -> Result<CopyFolderTreeResult, DomainError> {
let row = sqlx::query_as::<_, (String, i64, i64)>(
"SELECT new_root_id, folders_copied, files_copied \
FROM storage.copy_folder_tree($1::uuid, $2::uuid, $3)",
)
.bind(source_folder_id)
.bind(&target_parent_id)
.bind(&dest_name)
.fetch_one(self.pool.as_ref())
.await
.map_err(|e| {
// Map PG P0002 (no_data_found) to NotFound
if let sqlx::Error::Database(ref db_err) = e {
if db_err.code().as_deref() == Some("P0002") {
return DomainError::not_found("Folder", source_folder_id);
}
if db_err.code().as_deref() == Some("23505") {
return DomainError::already_exists(
"Folder",
"A folder with that name already exists in the target".to_string(),
);
}
}
DomainError::internal_error(
"FileBlobWrite",
format!("copy_folder_tree: {e}"),
)
})?;
tracing::info!(
"📂 TREE COPY: {} folders + {} files (root: {}, zero-copy via dedup)",
row.1,
row.2,
&row.0[..8]
);
Ok(CopyFolderTreeResult {
new_root_folder_id: row.0,
folders_copied: row.1,
files_copied: row.2,
})
}
}
src/infrastructure/services/dedup_service.rs
+12 -4
View File
@@ -52,13 +52,20 @@ pub struct DedupService {
blob_root: PathBuf,
/// Root directory for temporary files during upload
temp_root: PathBuf,
/// PostgreSQL connection pool (dedup index in `storage.blobs`)
/// PostgreSQL connection pool (dedup index in `storage.blobs`) — primary,
/// used by request-path operations (store_bytes, store_from_file, etc.).
pool: Arc<PgPool>,
/// Isolated maintenance pool for long-running operations
/// (verify_integrity, garbage_collect) that must never starve the primary.
maintenance_pool: Arc<PgPool>,
}
impl DedupService {
/// Create a new dedup service backed by PostgreSQL.
pub fn new(storage_root: &Path, pool: Arc<PgPool>) -> Self {
///
/// * `pool` — primary pool for request-path operations.
/// * `maintenance_pool` — isolated pool for verify_integrity / garbage_collect.
pub fn new(storage_root: &Path, pool: Arc<PgPool>, maintenance_pool: Arc<PgPool>) -> Self {
let blob_root = storage_root.join(".blobs");
let temp_root = storage_root.join(".dedup_temp");
@@ -66,6 +73,7 @@ impl DedupService {
blob_root,
temp_root,
pool,
maintenance_pool,
}
}
@@ -675,7 +683,7 @@ impl DedupService {
let mut row_stream = sqlx::query_as::<_, (String, i64)>(
"SELECT hash, size FROM storage.blobs ORDER BY hash",
)
.fetch(self.pool.as_ref());
.fetch(self.maintenance_pool.as_ref());
let mut total = 0usize;
let mut corrupted = Vec::<String>::new();
@@ -782,7 +790,7 @@ impl DedupService {
let mut orphan_stream = sqlx::query_as::<_, (String, i64)>(
"DELETE FROM storage.blobs WHERE ref_count = 0 RETURNING hash, size",
)
.fetch(self.pool.as_ref());
.fetch(self.maintenance_pool.as_ref());
let mut deleted_count = 0u64;
let mut deleted_bytes = 0u64;
src/interfaces/api/handlers/webdav_handler.rs
+35 -39
View File
@@ -1039,7 +1039,6 @@ async fn handle_copy(
// Get services from state
let file_retrieval_service = &state.applications.file_retrieval_service;
let _file_upload_service = &state.applications.file_upload_service;
let folder_service = &state.applications.folder_service;
// Check if destination already exists (for Overwrite header compliance)
@@ -1076,48 +1075,45 @@ async fn handle_copy(
""
};
// For now, just create a new folder and copy files individually
// In a real implementation, we would have a dedicated copy_folder service method
let create_dto = crate::application::dtos::folder_dto::CreateFolderDto {
name: dest_folder_name.to_string(),
parent_id: if dest_parent_path.is_empty() {
None
} else {
// Try to get the parent folder ID from its path
match folder_service.get_folder_by_path(dest_parent_path).await {
Ok(parent) => Some(parent.id),
Err(_) => None, // If not found, use root
}
},
let target_parent_id = if dest_parent_path.is_empty() {
None
} else {
match folder_service.get_folder_by_path(dest_parent_path).await {
Ok(parent) => Some(parent.id),
Err(_) => None,
}
};
let _new_folder = folder_service
.create_folder(create_dto)
.await
.map_err(|e| {
AppError::internal_error(format!("Failed to create destination folder: {}", e))
})?;
if recursive {
// Copy files via zero-copy dedup (only increments blob ref_count)
let files = file_retrieval_service
.list_files(Some(&folder.id))
.await
.map_err(|e| AppError::internal_error(format!("Failed to list files: {}", e)))?;
// Atomic recursive copy: single SQL function call copies the entire
// folder tree (all sub-folders + all files) with zero-copy dedup.
// O(depth) folder INSERTs + 1 batch file INSERT + 1 batch ref_count UPDATE.
let file_management_service = &state.applications.file_management_service;
let new_folder_id = Some(_new_folder.id.clone());
for file in files {
file_management_service
.copy_file(&file.id, new_folder_id.clone())
.await
.map_err(|e| {
AppError::internal_error(format!(
"Failed to copy file {}: {}",
file.name, e
))
})?;
}
file_management_service
.copy_folder_tree(
&folder.id,
target_parent_id,
Some(dest_folder_name.to_string()),
)
.await
.map_err(|e| {
AppError::internal_error(format!("Failed to copy folder tree: {}", e))
})?;
} else {
// Depth: 0 — create empty folder only (no sub-folder or file copy)
let create_dto = crate::application::dtos::folder_dto::CreateFolderDto {
name: dest_folder_name.to_string(),
parent_id: target_parent_id,
};
folder_service
.create_folder(create_dto)
.await
.map_err(|e| {
AppError::internal_error(format!(
"Failed to create destination folder: {}",
e
))
})?;
}
} else {
// Copy file — use zero-copy dedup (only increments blob ref_count, no content loaded)
src/main.rs
+8 -8
View File
@@ -36,7 +36,7 @@ use oxicloud::infrastructure;
use oxicloud::interfaces;
use common::di::AppServiceFactory;
use infrastructure::db::create_database_pool;
use infrastructure::db::create_database_pools;
use interfaces::{create_api_routes, create_public_api_routes, web::create_web_routes};
#[tokio::main]
@@ -65,12 +65,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
std::fs::create_dir_all(&locales_path).expect("Failed to create locales directory");
}
// Initialize database pool if auth is enabled
let db_pool = if config.features.enable_auth {
match create_database_pool(&config).await {
Ok(pool) => {
tracing::info!("PostgreSQL database pool initialized successfully");
Some(Arc::new(pool))
// Initialize database pools if auth is enabled
let db_pools = if config.features.enable_auth {
match create_database_pools(&config).await {
Ok(pools) => {
tracing::info!("PostgreSQL database pools initialized successfully");
Some(pools)
}
Err(e) => {
// SECURITY: fail-closed. If auth is required but the database
@@ -89,7 +89,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Build all services via the factory
let factory = AppServiceFactory::with_config(storage_path, locales_path, config.clone());
let app_state = factory.build_app_state(db_pool).await
let app_state = factory.build_app_state(db_pools).await
.expect("Failed to build application state. If running in Docker, ensure the storage volume is writable by the oxicloud user (UID 1001)");
// Wrap in Arc so that Axum clones a single refcount per request