761d159a92
- Replace whole-file SHA-256 dedup with FastCDC 2020 content-defined chunking (min 64KB, avg 256KB, max 1MB) + BLAKE3 hashing - Add chunk_manifests table (file_hash → chunk_hashes[] + chunk_sizes[]) - Add put_blob_from_bytes to BlobStorageBackend trait (all 7 backends) - 3-phase store_chunks pipeline: Phase 0: batch-check existing chunks (single PG query) Phase 1: selective disk read (skip existing chunks entirely) Phase 2: parallel upload with buffer_unordered(8) - CDC-aware read_blob_stream and read_blob_range_stream with legacy fallback - Transactional manifest + chunk ref-count cascade on remove_reference - 12 CDC tests (determinism, reassembly, contiguity, sub-file dedup, etc.) - Update deduplication.md to reflect new architecture
212 lines
6.8 KiB
Rust
212 lines
6.8 KiB
Rust
//! `MigrationBlobBackend` — decorator that enables zero-downtime migration
|
|
//! between blob storage backends.
|
|
//!
|
|
//! During a migration the decorator writes to the **target** backend and reads
|
|
//! from **target-first-then-source** (dual-read). A background job
|
|
//! (see `migration_job.rs`) copies remaining blobs in the background.
|
|
|
|
use std::future::Future;
|
|
use std::path::{Path, PathBuf};
|
|
use std::pin::Pin;
|
|
use std::sync::Arc;
|
|
|
|
use bytes::Bytes;
|
|
use chrono::{DateTime, Utc};
|
|
use serde::Serialize;
|
|
use tokio::sync::RwLock;
|
|
|
|
use crate::application::ports::blob_storage_ports::{
|
|
BlobStorageBackend, BlobStream, StorageHealthStatus,
|
|
};
|
|
use crate::common::errors::DomainError;
|
|
|
|
// ── Migration state ────────────────────────────────────────────────
|
|
|
|
/// Progress of an ongoing (or completed) backend migration.
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub struct MigrationState {
|
|
pub status: MigrationStatus,
|
|
pub total_blobs: u64,
|
|
pub migrated_blobs: u64,
|
|
pub migrated_bytes: u64,
|
|
pub failed_blobs: Vec<String>,
|
|
pub started_at: Option<DateTime<Utc>>,
|
|
pub completed_at: Option<DateTime<Utc>>,
|
|
}
|
|
|
|
impl Default for MigrationState {
|
|
fn default() -> Self {
|
|
Self {
|
|
status: MigrationStatus::Idle,
|
|
total_blobs: 0,
|
|
migrated_blobs: 0,
|
|
migrated_bytes: 0,
|
|
failed_blobs: Vec::new(),
|
|
started_at: None,
|
|
completed_at: None,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Status of the migration job.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum MigrationStatus {
|
|
Idle,
|
|
Running,
|
|
Paused,
|
|
Completed,
|
|
Failed,
|
|
}
|
|
|
|
// ── MigrationBlobBackend ───────────────────────────────────────────
|
|
|
|
/// A `BlobStorageBackend` decorator that proxies requests to a *source*
|
|
/// (old) and *target* (new) backend, enabling live migration.
|
|
pub struct MigrationBlobBackend {
|
|
source: Arc<dyn BlobStorageBackend>,
|
|
target: Arc<dyn BlobStorageBackend>,
|
|
state: Arc<RwLock<MigrationState>>,
|
|
}
|
|
|
|
impl MigrationBlobBackend {
|
|
pub fn new(
|
|
source: Arc<dyn BlobStorageBackend>,
|
|
target: Arc<dyn BlobStorageBackend>,
|
|
state: Arc<RwLock<MigrationState>>,
|
|
) -> Self {
|
|
Self {
|
|
source,
|
|
target,
|
|
state,
|
|
}
|
|
}
|
|
|
|
pub fn state(&self) -> &Arc<RwLock<MigrationState>> {
|
|
&self.state
|
|
}
|
|
|
|
pub fn source(&self) -> &Arc<dyn BlobStorageBackend> {
|
|
&self.source
|
|
}
|
|
|
|
pub fn target(&self) -> &Arc<dyn BlobStorageBackend> {
|
|
&self.target
|
|
}
|
|
}
|
|
|
|
/// Boxed future alias (same as in the trait module).
|
|
type BoxFut<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
|
|
|
impl BlobStorageBackend for MigrationBlobBackend {
|
|
fn initialize(&self) -> BoxFut<'_, Result<(), DomainError>> {
|
|
Box::pin(async move {
|
|
self.target.initialize().await?;
|
|
// Source is already initialised; call anyway for idempotency.
|
|
self.source.initialize().await?;
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// Writes go to **target** only.
|
|
fn put_blob(&self, hash: &str, source_path: &Path) -> BoxFut<'_, Result<u64, DomainError>> {
|
|
let hash = hash.to_string();
|
|
let path = source_path.to_path_buf();
|
|
Box::pin(async move { self.target.put_blob(&hash, &path).await })
|
|
}
|
|
|
|
/// Writes bytes to **target** only.
|
|
fn put_blob_from_bytes(&self, hash: &str, data: Bytes) -> BoxFut<'_, Result<u64, DomainError>> {
|
|
let hash = hash.to_string();
|
|
Box::pin(async move { self.target.put_blob_from_bytes(&hash, data).await })
|
|
}
|
|
|
|
/// Read from target first; fall back to source.
|
|
fn get_blob_stream(&self, hash: &str) -> BoxFut<'_, Result<BlobStream, DomainError>> {
|
|
let hash = hash.to_string();
|
|
Box::pin(async move {
|
|
match self.target.get_blob_stream(&hash).await {
|
|
Ok(stream) => Ok(stream),
|
|
Err(_) => self.source.get_blob_stream(&hash).await,
|
|
}
|
|
})
|
|
}
|
|
|
|
fn get_blob_range_stream(
|
|
&self,
|
|
hash: &str,
|
|
start: u64,
|
|
end: Option<u64>,
|
|
) -> BoxFut<'_, Result<BlobStream, DomainError>> {
|
|
let hash = hash.to_string();
|
|
Box::pin(async move {
|
|
match self.target.get_blob_range_stream(&hash, start, end).await {
|
|
Ok(stream) => Ok(stream),
|
|
Err(_) => self.source.get_blob_range_stream(&hash, start, end).await,
|
|
}
|
|
})
|
|
}
|
|
|
|
/// Delete from **both** backends (best-effort on source).
|
|
fn delete_blob(&self, hash: &str) -> BoxFut<'_, Result<(), DomainError>> {
|
|
let hash = hash.to_string();
|
|
Box::pin(async move {
|
|
self.target.delete_blob(&hash).await?;
|
|
// Best-effort on source — ignore errors (blob may already be gone).
|
|
let _ = self.source.delete_blob(&hash).await;
|
|
Ok(())
|
|
})
|
|
}
|
|
|
|
/// Exists in either backend.
|
|
fn blob_exists(&self, hash: &str) -> BoxFut<'_, Result<bool, DomainError>> {
|
|
let hash = hash.to_string();
|
|
Box::pin(async move {
|
|
if self.target.blob_exists(&hash).await? {
|
|
return Ok(true);
|
|
}
|
|
self.source.blob_exists(&hash).await
|
|
})
|
|
}
|
|
|
|
fn blob_size(&self, hash: &str) -> BoxFut<'_, Result<u64, DomainError>> {
|
|
let hash = hash.to_string();
|
|
Box::pin(async move {
|
|
match self.target.blob_size(&hash).await {
|
|
Ok(sz) => Ok(sz),
|
|
Err(_) => self.source.blob_size(&hash).await,
|
|
}
|
|
})
|
|
}
|
|
|
|
fn health_check(&self) -> BoxFut<'_, Result<StorageHealthStatus, DomainError>> {
|
|
Box::pin(async move {
|
|
let target_health = self.target.health_check().await?;
|
|
let source_health = self.source.health_check().await?;
|
|
Ok(StorageHealthStatus {
|
|
connected: target_health.connected && source_health.connected,
|
|
backend_type: format!(
|
|
"migration({} → {})",
|
|
source_health.backend_type, target_health.backend_type
|
|
),
|
|
message: format!(
|
|
"Source: {} | Target: {}",
|
|
source_health.message, target_health.message
|
|
),
|
|
available_bytes: target_health.available_bytes,
|
|
})
|
|
})
|
|
}
|
|
|
|
fn backend_type(&self) -> &'static str {
|
|
"migration"
|
|
}
|
|
|
|
fn local_blob_path(&self, hash: &str) -> Option<PathBuf> {
|
|
// Prefer target, fall back to source.
|
|
self.target
|
|
.local_blob_path(hash)
|
|
.or_else(|| self.source.local_blob_path(hash))
|
|
}
|
|
}
|