cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor(lifecycle hooks): simplify integration of new services

Browse Source 
* make more coherent lifecycles
  * remove specific implementation on different handlers (they do not need to know existence of ThumbnailSerice nor AudioMetadataService)
  * reduce risk of orphean objects
  * ensure additional services are correctly wired (ex: Thumbnail generation was not covering all upload cases)
  * more details on docs/architecture/file-and-blob-lifecycle.md :

```rust
// application/ports/file_lifecycle.rs
pub trait FileLifecycleHook {

    fn on_file_created(file_id, blob_hash, content_type, is_new_blob);
    fn on_file_updated(file_id, blob_hash, content_type);
    fn on_file_copied(file_id, blob_hash, content_type, source_id)
    fn on_file_deleted(file_id);
}

// application/ports/blob_lifecycle.rs
pub trait BlobLifecycleHook {

    fn on_blob_created(blob_hash, content_type);
    fn on_blob_deleted(blob_hash);
}
```
This commit is contained in:
Edouard Vanbelle
2026-05-22 13:10:47 +02:00
parent 9206669ee6
commit 73f0b0fa47
19 changed files with 842 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/infrastructure/repositories/pg/file_blob_write_repository.rs
+32 -9
View File
@@ -202,10 +202,11 @@ impl FileBlobWriteRepository {
Ok(new_hash.to_string())
}
}
impl FileWritePort for FileBlobWriteRepository {
async fn save_file_from_temp(
/// Like [`FileWritePort::save_file_from_temp`] but also returns whether the
/// blob was genuinely new (`true`) or a dedup hit (`false`).
/// Used by [`FileUploadService`] to pass `is_new_blob` to lifecycle hooks.
pub async fn save_file_from_temp_with_dedup(
&self,
name: String,
folder_id: Option<String>,
@@ -213,18 +214,16 @@ impl FileWritePort for FileBlobWriteRepository {
temp_path: &std::path::Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<File, DomainError> {
) -> Result<(File, bool), DomainError> {
let user_id = self.resolve_user_id(folder_id.as_deref()).await?;
// True streaming: pass pre-computed hash (or let dedup compute it).
// When hash is pre-computed, zero extra disk reads.
let dedup_result = self
.dedup
.store_from_file(temp_path, Some(content_type.clone()), pre_computed_hash)
.await?;
let is_new_blob = !dedup_result.was_deduplicated();
let blob_hash = dedup_result.hash().to_string();
// Insert file metadata — if this fails, compensate by removing the blob ref
let row = match sqlx::query_as::<_, (String, i64, i64)>(
r#"
INSERT INTO storage.files (name, folder_id, user_id, blob_hash, size, mime_type)
@@ -275,7 +274,7 @@ impl FileWritePort for FileBlobWriteRepository {
);
let folder_path = self.lookup_folder_path(folder_id.as_deref()).await?;
Self::row_to_file(
let file = Self::row_to_file(
row.0,
name,
folder_id,
@@ -285,8 +284,32 @@ impl FileWritePort for FileBlobWriteRepository {
row.1,
row.2,
Some(user_id),
blob_hash.clone(),
blob_hash,
)?;
Ok((file, is_new_blob))
}
}
impl FileWritePort for FileBlobWriteRepository {
async fn save_file_from_temp(
&self,
name: String,
folder_id: Option<String>,
content_type: String,
temp_path: &std::path::Path,
size: u64,
pre_computed_hash: Option<String>,
) -> Result<File, DomainError> {
self.save_file_from_temp_with_dedup(
name,
folder_id,
content_type,
temp_path,
size,
pre_computed_hash,
)
.await
.map(|(file, _)| file)
}
async fn move_file(
src/infrastructure/services/audio_metadata_service.rs
+172
View File
@@ -6,6 +6,7 @@ use std::sync::Arc;
use tracing::{info, warn};
use uuid::Uuid;
use crate::application::ports::file_lifecycle::FileLifecycleHook;
use crate::common::errors::DomainError;
#[derive(Debug, FromRow)]
@@ -231,6 +232,101 @@ impl AudioMetadataService {
failed,
})
}
/// Copy audio metadata from an existing file that shares the same blob.
///
/// Used when `is_new_blob=false` (copy/dedup hit): instead of re-parsing
/// the blob, clone the existing metadata row for `new_file_id`. Falls back
/// Clones audio metadata from a known source file, falling back to
/// [`clone_or_extract_background`] if the source has not been processed yet.
pub fn clone_from_source_background(
service: Arc<Self>,
new_file_id: Uuid,
source_file_id: Uuid,
blob_hash: String,
) {
tokio::spawn(async move {
let result = sqlx::query(
r#"
INSERT INTO audio.file_metadata
(file_id, title, artist, album, album_artist, genre,
track_number, disc_number, year, duration_secs, format)
SELECT $1, title, artist, album, album_artist, genre,
track_number, disc_number, year, duration_secs, format
FROM audio.file_metadata
WHERE file_id = $2
ON CONFLICT (file_id) DO NOTHING
"#,
)
.bind(new_file_id)
.bind(source_file_id)
.execute(&*service.pool)
.await;
match result {
Ok(r) if r.rows_affected() > 0 => {
info!(
"Cloned audio metadata from {} to {}",
source_file_id, new_file_id
);
}
Ok(_) => {
// Source not yet processed — fall back to blob-hash lookup or extraction.
Self::clone_or_extract_background(service, new_file_id, blob_hash);
}
Err(e) => {
warn!(
"Failed to clone audio metadata from {} to {}: {}",
source_file_id, new_file_id, e
);
}
}
});
}
/// to full extraction if no existing row is found (race: original not yet
/// processed).
pub fn clone_or_extract_background(service: Arc<Self>, new_file_id: Uuid, blob_hash: String) {
tokio::spawn(async move {
let rows_inserted = sqlx::query(
r#"
INSERT INTO audio.file_metadata
(file_id, title, artist, album, album_artist, genre,
track_number, disc_number, year, duration_secs, format)
SELECT $1, title, artist, album, album_artist, genre,
track_number, disc_number, year, duration_secs, format
FROM audio.file_metadata am
JOIN storage.files sf ON sf.id = am.file_id
WHERE sf.blob_hash = $2
LIMIT 1
ON CONFLICT (file_id) DO NOTHING
"#,
)
.bind(new_file_id)
.bind(&blob_hash)
.execute(&*service.pool)
.await;
match rows_inserted {
Ok(result) if result.rows_affected() > 0 => {
info!("Cloned audio metadata for file {}", new_file_id);
}
Ok(_) => {
// No existing metadata found — original not yet processed; fall back.
let file_path = service.blob_path(&blob_hash);
if let Err(e) = service.extract_and_save(&new_file_id, &file_path).await {
warn!(
"Failed to extract audio metadata for {}: {}",
new_file_id, e
);
}
}
Err(e) => {
warn!("Failed to clone audio metadata for {}: {}", new_file_id, e);
}
}
});
}
}
/// Extracted audio metadata fields transferred from the blocking thread.
@@ -252,3 +348,79 @@ pub struct MetadataExtractionResult {
pub processed: usize,
pub failed: usize,
}
// ─── FileLifecycleHook ───────────────────────────────────────────────────────
impl FileLifecycleHook for AudioMetadataService {
fn on_file_created(
&self,
file_id: &str,
blob_hash: &str,
content_type: &str,
is_new_blob: bool,
) {
if !Self::is_audio_file(content_type) {
return;
}
let Ok(uuid) = file_id.parse::<Uuid>() else {
warn!("on_file_created: invalid file_id UUID: {}", file_id);
return;
};
let service = Arc::new(Self {
pool: self.pool.clone(),
blob_root: self.blob_root.clone(),
});
if is_new_blob {
Self::spawn_extraction_background(service, uuid, self.blob_path(blob_hash));
} else {
Self::clone_or_extract_background(service, uuid, blob_hash.to_string());
}
}
fn on_file_copied(
&self,
file_id: &str,
blob_hash: &str,
content_type: &str,
source_file_id: &str,
) {
if !Self::is_audio_file(content_type) {
return;
}
let Ok(uuid) = file_id.parse::<Uuid>() else {
warn!("on_file_copied: invalid file_id UUID: {}", file_id);
return;
};
let Ok(source_uuid) = source_file_id.parse::<Uuid>() else {
warn!(
"on_file_copied: invalid source_file_id UUID: {}",
source_file_id
);
return;
};
let service = Arc::new(Self {
pool: self.pool.clone(),
blob_root: self.blob_root.clone(),
});
Self::clone_from_source_background(service, uuid, source_uuid, blob_hash.to_string());
}
fn on_file_updated(&self, file_id: &str, blob_hash: &str, content_type: &str) {
if !Self::is_audio_file(content_type) {
return;
}
let Ok(uuid) = file_id.parse::<Uuid>() else {
warn!("on_file_updated: invalid file_id UUID: {}", file_id);
return;
};
let service = Arc::new(Self {
pool: self.pool.clone(),
blob_root: self.blob_root.clone(),
});
Self::spawn_extraction_with_delete_background(service, uuid, self.blob_path(blob_hash));
}
fn on_file_deleted(&self, _file_id: &str) {
// audio.file_metadata has ON DELETE CASCADE on file_id — DB handles cleanup.
}
}
src/infrastructure/services/dedup_service.rs
+20 -34
View File
@@ -44,11 +44,12 @@ use std::sync::Arc;
use tokio::fs;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use crate::application::ports::blob_lifecycle::{BlobCreationHook, BlobDeletionHook};
use crate::application::ports::blob_lifecycle::BlobLifecycleHook;
use crate::application::ports::blob_storage_ports::BlobStorageBackend;
use crate::application::ports::dedup_ports::{
BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto,
};
use crate::application::services::blob_lifecycle_service::BlobLifecycleService;
use crate::domain::errors::{DomainError, ErrorKind};
// ── CDC Constants ────────────────────────────────────────────────────────────
@@ -84,10 +85,8 @@ pub struct DedupService {
/// Isolated maintenance pool for long-running operations
/// (verify_integrity, garbage_collect) that must never starve the primary.
maintenance_pool: Arc<PgPool>,
/// Hooks notified when a genuinely new blob is stored (no dedup hit).
blob_creation_hooks: Vec<Arc<dyn BlobCreationHook>>,
/// Hooks notified when a blob's ref_count reaches zero and it is deleted.
blob_hooks: Vec<Arc<dyn BlobDeletionHook>>,
/// Single lifecycle dispatcher — fired on blob created / deleted.
blob_lifecycle: Option<Arc<BlobLifecycleService>>,
}
impl DedupService {
@@ -105,36 +104,25 @@ impl DedupService {
backend,
pool,
maintenance_pool,
blob_creation_hooks: vec![],
blob_hooks: vec![],
blob_lifecycle: None,
}
}
/// Register a [`BlobCreationHook`] to be called whenever a genuinely new
/// blob is stored. Hooks are called in registration order.
pub fn add_blob_creation_hook(mut self, hook: Arc<dyn BlobCreationHook>) -> Self {
self.blob_creation_hooks.push(hook);
/// Registers the blob lifecycle dispatcher (thumbnail cleanup, …).
pub fn with_blob_lifecycle(mut self, lifecycle: Arc<BlobLifecycleService>) -> Self {
self.blob_lifecycle = Some(lifecycle);
self
}
/// Register a [`BlobDeletionHook`] to be called whenever a blob's
/// ref_count reaches zero. Hooks are called in registration order.
pub fn add_blob_hook(mut self, hook: Arc<dyn BlobDeletionHook>) -> Self {
self.blob_hooks.push(hook);
self
}
/// Fire all registered creation hooks for a new blob.
async fn fire_blob_creation_hooks(&self, hash: &str, content_type: Option<&str>) {
for hook in &self.blob_creation_hooks {
hook.on_blob_created(hash, content_type).await;
fn fire_blob_creation_hooks(&self, hash: &str, content_type: Option<&str>) {
if let Some(lc) = &self.blob_lifecycle {
lc.on_blob_created(hash, content_type);
}
}
/// Fire all registered hooks for a deleted blob.
async fn fire_blob_hooks(&self, hash: &str) {
for hook in &self.blob_hooks {
hook.on_blob_deleted(hash).await;
fn fire_blob_hooks(&self, hash: &str) {
if let Some(lc) = &self.blob_lifecycle {
lc.on_blob_deleted(hash);
}
}
@@ -152,8 +140,7 @@ impl DedupService {
backend: Arc::new(LocalBlobBackend::new(Path::new("/tmp/oxicloud_stub_blobs"))),
pool: stub_pool.clone(),
maintenance_pool: stub_pool,
blob_creation_hooks: vec![],
blob_hooks: vec![],
blob_lifecycle: None,
}
}
@@ -371,8 +358,7 @@ impl DedupService {
chunk_hashes.len()
);
self.fire_blob_creation_hooks(&file_hash, content_type.as_deref())
.await;
self.fire_blob_creation_hooks(&file_hash, content_type.as_deref());
Ok(DedupResultDto::NewBlob {
hash: file_hash,
@@ -813,7 +799,7 @@ impl DedupService {
}
// Bug 4 fix: notify hooks — e.g. thumbnail cleanup keyed by file_hash
self.fire_blob_hooks(file_hash).await;
self.fire_blob_hooks(file_hash);
tracing::info!(
"MANIFEST DELETED: {} ({} chunks, {} orphan chunks removed)",
@@ -893,7 +879,7 @@ impl DedupService {
}
// Bug 3 fix: notify hooks — e.g. thumbnail cleanup keyed by hash
self.fire_blob_hooks(hash).await;
self.fire_blob_hooks(hash);
tracing::info!("BLOB DELETED: {} (no more references)", &hash[..12]);
Ok(true)
@@ -1000,7 +986,7 @@ impl DedupService {
if let Err(e) = self.backend.delete_blob(hash).await {
tracing::warn!("cleanup_if_orphaned: disk delete failed for {short}: {e}");
}
self.fire_blob_hooks(hash).await;
self.fire_blob_hooks(hash);
tracing::info!("cleanup_if_orphaned: removed orphaned blob {short}");
}
}
@@ -1471,7 +1457,7 @@ impl DedupService {
if let Err(e) = self.backend.delete_blob(hash).await {
tracing::warn!("Failed to delete orphan blob {hash}: {e}");
}
self.fire_blob_hooks(hash).await;
self.fire_blob_hooks(hash);
total_bytes += *size as u64;
}
total_deleted += batch.len() as u64;
src/infrastructure/services/thumbnail_service.rs
+82 -64
View File
@@ -984,23 +984,12 @@ impl ThumbnailService {
}
}
// ─── BlobDeletionHook ────────────────────────────────────────────────────────
// ─── FileLifecycleHook + BlobLifecycleHook ───────────────────────────────────
impl crate::application::ports::blob_lifecycle::BlobDeletionHook for ThumbnailService {
fn on_blob_deleted<'a>(
&'a self,
blob_hash: &'a str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
Box::pin(async move { self.delete_blob_thumbnails(blob_hash).await })
}
}
// ─── FileUpdatedHook ─────────────────────────────────────────────────────────
/// Wires thumbnail invalidation + regeneration into the file-update lifecycle.
/// Wires all thumbnail side-effects into the file and blob lifecycle.
///
/// Registered on [`FileUploadService`] during DI. Fires whenever a file's blob
/// is replaced (WebDAV PUT overwrite, WOPI PutFile, Nextcloud chunked upload).
/// Registered once on both [`FileLifecycleService`] and [`BlobLifecycleService`]
/// during DI. Handles thumbnail generation, invalidation, and cleanup.
pub struct ThumbnailRefreshHook {
thumbnail: Arc<ThumbnailService>,
dedup: Arc<DedupService>,
@@ -1012,54 +1001,70 @@ impl ThumbnailRefreshHook {
}
}
impl crate::application::ports::file_lifecycle::FileUpdatedHook for ThumbnailRefreshHook {
fn on_file_updated<'a>(
&'a self,
file_id: &'a str,
blob_hash: &'a str,
content_type: &'a str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
Box::pin(async move {
if !ThumbnailService::is_supported_image(content_type) {
return;
}
if let Err(e) = self.thumbnail.delete_thumbnails(file_id).await {
impl crate::application::ports::file_lifecycle::FileLifecycleHook for ThumbnailRefreshHook {
fn on_file_created(
&self,
file_id: &str,
blob_hash: &str,
content_type: &str,
is_new_blob: bool,
) {
// Blob-hash thumbnail already exists on disk when is_new_blob=false — skip.
if !is_new_blob || !ThumbnailService::is_supported_image(content_type) {
return;
}
Self::spawn_thumbnail_generation(
self.thumbnail.clone(),
self.dedup.clone(),
file_id.to_string(),
blob_hash.to_string(),
);
}
fn on_file_copied(
&self,
_file_id: &str,
_blob_hash: &str,
_content_type: &str,
_source_file_id: &str,
) {
// Thumbnails are keyed by blob_hash on disk — the copy shares them automatically.
}
fn on_file_updated(&self, file_id: &str, blob_hash: &str, content_type: &str) {
if !ThumbnailService::is_supported_image(content_type) {
return;
}
let thumbnail = self.thumbnail.clone();
let file_id = file_id.to_string();
let blob_hash = blob_hash.to_string();
let dedup = self.dedup.clone();
tokio::spawn(async move {
if let Err(e) = thumbnail.delete_thumbnails(&file_id).await {
tracing::warn!(
"Failed to invalidate thumbnail cache for {}: {}",
file_id,
e
);
}
Self::spawn_thumbnail_generation(
self.thumbnail.clone(),
self.dedup.clone(),
file_id.to_string(),
blob_hash.to_string(),
);
})
Self::spawn_thumbnail_generation(thumbnail, dedup, file_id, blob_hash);
});
}
fn on_file_deleted(&self, file_id: &str) {
let thumbnail = self.thumbnail.clone();
let file_id = file_id.to_string();
tokio::spawn(async move {
if let Err(e) = thumbnail.delete_thumbnails(&file_id).await {
tracing::warn!("Failed to delete thumbnails for file {}: {}", file_id, e);
}
});
}
}
impl crate::application::ports::file_lifecycle::FileCreatedHook for ThumbnailRefreshHook {
fn on_file_created<'a>(
&'a self,
file_id: &'a str,
blob_hash: &'a str,
content_type: &'a str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
Box::pin(async move {
if !ThumbnailService::is_supported_image(content_type) {
return;
}
Self::spawn_thumbnail_generation(
self.thumbnail.clone(),
self.dedup.clone(),
file_id.to_string(),
blob_hash.to_string(),
);
})
}
}
// BlobLifecycleHook is implemented on ThumbnailService (not ThumbnailRefreshHook)
// to avoid a circular Arc: DedupService→BlobLifecycleService→ThumbnailRefreshHook→DedupService.
// ThumbnailService does not hold DedupService so no cycle exists.
impl ThumbnailRefreshHook {
fn spawn_thumbnail_generation(
@@ -1085,18 +1090,31 @@ impl ThumbnailRefreshHook {
}
}
// ─── FileDeletedHook ─────────────────────────────────────────────────────────
// ─── BlobLifecycleHook ───────────────────────────────────────────────────────
impl crate::application::ports::file_lifecycle::FileDeletedHook for ThumbnailService {
fn on_file_deleted<'a>(
&'a self,
file_id: &'a str,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>> {
Box::pin(async move {
if let Err(e) = self.delete_thumbnails(file_id).await {
tracing::warn!("Failed to delete thumbnails for file {}: {}", file_id, e);
impl crate::application::ports::blob_lifecycle::BlobLifecycleHook for ThumbnailService {
fn on_blob_created(&self, _blob_hash: &str, _content_type: Option<&str>) {
// Thumbnail generation is driven by file-level events (on_file_created).
}
fn on_blob_deleted(&self, blob_hash: &str) {
// delete_blob_thumbnails only needs thumbnails_root — capture it to avoid Arc cycle.
let root = self.thumbnails_root.clone();
let blob_hash = blob_hash.to_string();
tokio::spawn(async move {
for size in ThumbnailSize::all() {
let path = root
.join(size.dir_name())
.join(format!("{}.jpg", &blob_hash));
if tokio::fs::metadata(&path).await.is_ok() {
let _ = tokio::fs::remove_file(&path).await;
}
}
})
tracing::debug!(
"🗑️ Deleted blob thumbnails for hash: {}…",
&blob_hash[..blob_hash.len().min(12)]
);
});
}
}