feat(dedup): CDC sub-file deduplication with FastCDC + parallel chunk storage + dedup skip

- Replace whole-file SHA-256 dedup with FastCDC 2020 content-defined chunking
  (min 64KB, avg 256KB, max 1MB) + BLAKE3 hashing
- Add chunk_manifests table (file_hash → chunk_hashes[] + chunk_sizes[])
- Add put_blob_from_bytes to BlobStorageBackend trait (all 7 backends)
- 3-phase store_chunks pipeline:
  Phase 0: batch-check existing chunks (single PG query)
  Phase 1: selective disk read (skip existing chunks entirely)
  Phase 2: parallel upload with buffer_unordered(8)
- CDC-aware read_blob_stream and read_blob_range_stream with legacy fallback
- Transactional manifest + chunk ref-count cascade on remove_reference
- 12 CDC tests (determinism, reassembly, contiguity, sub-file dedup, etc.)
- Update deduplication.md to reflect new architecture
This commit is contained in:
Diocrafts
2026-04-14 23:17:39 +02:00
parent cd3733b459
commit 761d159a92
15 changed files with 1988 additions and 530 deletions
@@ -4,6 +4,7 @@
//! Wasabi, and any other service that implements the S3 API.
use aws_sdk_s3::primitives::ByteStream;
use bytes::Bytes;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use tokio::fs;
@@ -157,6 +158,48 @@ impl BlobStorageBackend for S3BlobBackend {
})
}
fn put_blob_from_bytes(
&self,
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let key = Self::object_key(&hash);
let size = data.len() as u64;
// Idempotent: skip if already exists
if self
.client
.head_object()
.bucket(&self.bucket)
.key(&key)
.send()
.await
.is_ok()
{
return Ok(size);
}
let body = ByteStream::from(data);
self.client
.put_object()
.bucket(&self.bucket)
.key(&key)
.body(body)
.send()
.await
.map_err(|e| {
DomainError::internal_error(
"S3",
format!("Failed to upload blob {}: {}", hash, e),
)
})?;
Ok(size)
})
}
fn get_blob_stream(
&self,
hash: &str,