feat(dedup): CDC sub-file deduplication with FastCDC + parallel chunk storage + dedup skip
- Replace whole-file SHA-256 dedup with FastCDC 2020 content-defined chunking (min 64KB, avg 256KB, max 1MB) + BLAKE3 hashing - Add chunk_manifests table (file_hash → chunk_hashes[] + chunk_sizes[]) - Add put_blob_from_bytes to BlobStorageBackend trait (all 7 backends) - 3-phase store_chunks pipeline: Phase 0: batch-check existing chunks (single PG query) Phase 1: selective disk read (skip existing chunks entirely) Phase 2: parallel upload with buffer_unordered(8) - CDC-aware read_blob_stream and read_blob_range_stream with legacy fallback - Transactional manifest + chunk ref-count cascade on remove_reference - 12 CDC tests (determinism, reassembly, contiguity, sub-file dedup, etc.) - Update deduplication.md to reflect new architecture
This commit is contained in:
@@ -12,6 +12,7 @@ use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use bytes::Bytes;
|
||||
use lru::LruCache;
|
||||
use std::num::NonZeroUsize;
|
||||
use tokio::fs;
|
||||
@@ -151,6 +152,37 @@ impl BlobStorageBackend for CachedBlobBackend {
|
||||
})
|
||||
}
|
||||
|
||||
fn put_blob_from_bytes(
|
||||
&self,
|
||||
hash: &str,
|
||||
data: Bytes,
|
||||
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
||||
let inner = self.inner.clone();
|
||||
let hash = hash.to_string();
|
||||
let self_ref = CachedRef {
|
||||
cache_dir: self.cache_dir.clone(),
|
||||
max_cache_bytes: self.max_cache_bytes,
|
||||
index: self.index.clone(),
|
||||
current_size: self.current_size.clone(),
|
||||
};
|
||||
Box::pin(async move {
|
||||
let size = inner.put_blob_from_bytes(&hash, data.clone()).await?;
|
||||
// Also cache locally (best-effort): write bytes to cache path
|
||||
let dest = self_ref.cached_path(&hash);
|
||||
if let Some(parent) = dest.parent() {
|
||||
let _ = fs::create_dir_all(parent).await;
|
||||
}
|
||||
let _ = fs::write(&dest, &data).await;
|
||||
let data_len = data.len() as u64;
|
||||
let mut idx = self_ref.index.lock().await;
|
||||
if let Some(old) = idx.put(hash, CacheEntry { size: data_len }) {
|
||||
self_ref.current_size.fetch_sub(old.size, Ordering::Relaxed);
|
||||
}
|
||||
self_ref.current_size.fetch_add(data_len, Ordering::Relaxed);
|
||||
Ok(size)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_blob_stream(
|
||||
&self,
|
||||
hash: &str,
|
||||
|
||||
Reference in New Issue
Block a user