Files
Oxicloud/src/infrastructure/services/local_blob_backend.rs
T

651 lines
25 KiB
Rust
Raw Normal View History

//! Local Filesystem Blob Backend — stores blobs under `.blobs/{prefix}/{hash}.blob`.
//!
//! This is the default backend and a direct extraction of the filesystem I/O
//! that previously lived inside `DedupService`.
use std::path::{Path, PathBuf};
use std::pin::Pin;
use tokio::fs::{self, File};
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
use tokio_util::io::ReaderStream;
use bytes::Bytes;
use crate::application::ports::blob_storage_ports::{
BlobStorageBackend, BlobStream, StorageHealthStatus,
};
use crate::domain::errors::{DomainError, ErrorKind};
/// Fsync the directory containing `child_path` so a preceding rename
/// or create on `child_path` becomes durable across power loss.
///
/// On Linux this issues `fsync(2)` on the directory file descriptor —
/// the canonical "make the dirent change durable" idiom. macOS does
/// the same but only persists to the disk controller (true persistence
/// would need `fcntl(F_FULLFSYNC)`, which tokio doesn't expose). On
/// Windows, opening a directory needs `FILE_FLAG_BACKUP_SEMANTICS` that
/// tokio's `File::open` doesn't set; that platform falls through to
/// `Ok(())` after a debug log.
///
/// Best-effort by design: a failure here is logged but does NOT fail
/// the upload, because the blob file itself was just `sync_all`'d and
/// is durable on its own. Worst case post-crash recovery: a rename
/// "reverts" to the un-renamed name (or stays renamed); the dedup-GC
/// cleanup pass handles either side.
async fn fsync_parent_dir(child_path: &Path) {
let Some(parent) = child_path.parent() else {
return;
};
let parent = parent.to_owned();
// std::fs (synchronous) opens directories reliably on Linux/macOS;
// do it on the blocking pool so we don't park the tokio worker.
let result = tokio::task::spawn_blocking(move || -> std::io::Result<()> {
let dir = std::fs::File::open(&parent)?;
dir.sync_all()
})
.await;
match result {
Ok(Ok(())) => {}
Ok(Err(e)) => {
tracing::warn!(
error = %e,
path = %child_path.display(),
"Blob parent-dir fsync failed (rename durability not guaranteed)"
);
}
Err(e) => {
tracing::warn!(
error = %e,
path = %child_path.display(),
"Blob parent-dir fsync task join failed"
);
}
}
}
/// Chunk size for streaming file reads (256 KB).
const STREAM_CHUNK_SIZE: usize = 256 * 1024;
/// Max parallel blocking tasks for the [`fsync_paths_parallel`] sweep.
///
/// Concurrent fsyncs let journaling filesystems coalesce barriers (ext4
/// merges parallel fsyncs into shared journal commits), so a sweep over
/// thousands of chunk files costs a small fraction of issuing the same
/// fsyncs sequentially.
const SYNC_SWEEP_CONCURRENCY: usize = 16;
/// Fsync every path in `paths`, spread over up to
/// [`SYNC_SWEEP_CONCURRENCY`] blocking-pool tasks.
///
/// `strict` mirrors the two durability tiers already present in this
/// module: blob *files* must be durable (hard error on failure, like
/// `put_blob_from_bytes`), while *directory* fsyncs are best-effort
/// (logged warning, like [`fsync_parent_dir`]) — directories can't be
/// opened for fsync on every platform.
async fn fsync_paths_parallel(paths: Vec<PathBuf>, strict: bool) -> Result<(), DomainError> {
if paths.is_empty() {
return Ok(());
}
let group_size = paths.len().div_ceil(SYNC_SWEEP_CONCURRENCY);
let mut tasks = Vec::with_capacity(SYNC_SWEEP_CONCURRENCY);
for group in paths.chunks(group_size) {
let group = group.to_vec();
tasks.push(tokio::task::spawn_blocking(
move || -> Result<(), (PathBuf, std::io::Error)> {
for path in &group {
let result = std::fs::File::open(path).and_then(|f| f.sync_all());
if let Err(e) = result {
if strict {
return Err((path.clone(), e));
}
tracing::warn!(
error = %e,
path = %path.display(),
"Blob sync sweep: best-effort fsync failed"
);
}
}
Ok(())
},
));
}
for task in tasks {
task.await
.map_err(|e| DomainError::internal_error("Blob", format!("sync sweep join: {e}")))?
.map_err(|(path, e)| {
DomainError::internal_error(
"Blob",
format!("sync sweep fsync of {} failed: {e}", path.display()),
)
})?;
}
Ok(())
}
/// Create `blob_path` and write `data` into it.
///
/// Returns the open file handle so the caller decides the durability tier
/// (fsync now vs. deferred batch sync), or `None` when the blob already
/// existed (idempotent skip — content-addressed, so identical by definition).
async fn write_blob_bytes(blob_path: &Path, data: &Bytes) -> Result<Option<File>, DomainError> {
// One atomic O_CREAT|O_EXCL open replaces the old stat-then-create pair:
// `AlreadyExists` IS the idempotent skip (content-addressed names mean an
// existing file has identical content), saving a syscall + a blocking-pool
// dispatch on every new chunk of every upload.
let mut file = match fs::File::options()
.write(true)
.create_new(true)
.open(blob_path)
.await
{
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => return Ok(None),
Err(e) => {
return Err(DomainError::internal_error(
"Blob",
format!("Failed to create blob file: {}", e),
));
}
};
file.write_all(data).await.map_err(|e| {
DomainError::internal_error("Blob", format!("Failed to write blob from bytes: {}", e))
})?;
Ok(Some(file))
}
/// Bench-only public wrapper (feature = "bench") over the private chunk
/// writer so `examples/bench_storage_micro.rs` can A/B the open strategy.
#[cfg(feature = "bench")]
pub async fn write_blob_bytes_for_bench(
blob_path: &Path,
data: &Bytes,
) -> Result<Option<File>, DomainError> {
write_blob_bytes(blob_path, data).await
}
/// Compile-time lookup table for the 256 two-digit lowercase hex prefixes ("00"…"ff").
static HEX_PREFIXES: [&str; 256] = [
"00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0a", "0b", "0c", "0d", "0e", "0f",
"10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "1a", "1b", "1c", "1d", "1e", "1f",
"20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "2a", "2b", "2c", "2d", "2e", "2f",
"30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "3a", "3b", "3c", "3d", "3e", "3f",
"40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "4a", "4b", "4c", "4d", "4e", "4f",
"50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "5a", "5b", "5c", "5d", "5e", "5f",
"60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "6a", "6b", "6c", "6d", "6e", "6f",
"70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "7a", "7b", "7c", "7d", "7e", "7f",
"80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "8a", "8b", "8c", "8d", "8e", "8f",
"90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "9a", "9b", "9c", "9d", "9e", "9f",
"a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9", "aa", "ab", "ac", "ad", "ae", "af",
"b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9", "ba", "bb", "bc", "bd", "be", "bf",
"c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "ca", "cb", "cc", "cd", "ce", "cf",
"d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "da", "db", "dc", "dd", "de", "df",
"e0", "e1", "e2", "e3", "e4", "e5", "e6", "e7", "e8", "e9", "ea", "eb", "ec", "ed", "ee", "ef",
"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "fa", "fb", "fc", "fd", "fe", "ff",
];
/// Local filesystem blob backend.
///
/// Blobs are stored under `blob_root/{2-char-prefix}/{hash}.blob`.
/// Temporary upload staging uses `temp_root/`.
pub struct LocalBlobBackend {
blob_root: PathBuf,
temp_root: PathBuf,
/// Chunk read-ahead depth for CDC reassembly — see [`Self::new`].
read_prefetch: usize,
}
/// Default chunk-open read-ahead for the local backend (overrides the trait's
/// conservative `1`).
///
/// Benchmarked with `examples/bench_blob_prefetch` on SSD-class storage: a small
/// read-ahead is the sweet spot for the *disk-bound* read paths — localhost/LAN
/// downloads and, importantly, the internal blob reads that drain as fast as the
/// disk delivers (thumbnail render, transcode, ZIP export, content extraction),
/// all of which flow through `DedupService::stream_chunks`'s `buffered(N)`.
///
/// Measured median throughput vs the old sequential `N=1`:
/// warm disk-bound +11.8% (N=2) cold disk-bound +7.2% (N=2)
/// network-bound (throttled) ≈ 0% — the consumer, not the disk, is the cap
/// N=16 −4.4% warm — fan-out past a couple turns one sequential read into
/// competing random I/O over scattered content-addressed chunk files.
///
/// `2` deliberately captures most of that gain at the lowest fan-out, because
/// `buffered(N)` here overlaps the per-chunk `File::open` (cheap on local disk),
/// not the data read, so deeper queues buy little and risk seek contention on
/// the spinning disks we can't bench here. Operators tune it via
/// `OXICLOUD_LOCAL_READ_PREFETCH` (set `1` on seek-bound HDDs to restore the old
/// strictly-sequential behaviour; raise it on fast NVMe arrays).
const DEFAULT_LOCAL_READ_PREFETCH: usize = 2;
impl LocalBlobBackend {
/// Create a new local backend rooted at `storage_root`.
///
/// Blob files go under `{storage_root}/.blobs/`, temp files under
/// `{storage_root}/.dedup_temp/`.
pub fn new(storage_root: &Path) -> Self {
// Read-ahead depth: env override, else the benchmark-backed default.
// Clamped to ≥1 so a bogus `0` can't stall reads (buffered(0) would
// make no progress; `stream_chunks` also guards with `.max(1)`).
let read_prefetch = std::env::var("OXICLOUD_LOCAL_READ_PREFETCH")
.ok()
.and_then(|v| v.parse::<usize>().ok())
.map(|n| n.max(1))
.unwrap_or(DEFAULT_LOCAL_READ_PREFETCH);
Self {
blob_root: storage_root.join(".blobs"),
temp_root: storage_root.join(".dedup_temp"),
read_prefetch,
}
}
/// Compute the filesystem path for a blob hash.
pub fn blob_path(&self, hash: &str) -> PathBuf {
let prefix = &hash[0..2];
self.blob_root.join(prefix).join(format!("{}.blob", hash))
}
/// Return a reference to the blob root directory.
pub fn blob_root(&self) -> &Path {
&self.blob_root
}
}
impl BlobStorageBackend for LocalBlobBackend {
fn initialize(
&self,
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
Box::pin(async move {
fs::create_dir_all(&self.blob_root)
.await
.map_err(DomainError::from)?;
fs::create_dir_all(&self.temp_root)
.await
.map_err(DomainError::from)?;
// Create the 256 hash-prefix directories (00-ff)
for prefix in &HEX_PREFIXES {
fs::create_dir_all(self.blob_root.join(prefix))
.await
.map_err(DomainError::from)?;
}
Ok(())
})
}
fn put_blob(
&self,
hash: &str,
source_path: &Path,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
let source_path = source_path.to_owned();
Box::pin(async move {
let blob_path = self.blob_path(&hash);
let file_size = fs::metadata(&source_path)
.await
.map_err(|e| {
DomainError::internal_error(
"Blob",
format!("Failed to stat source file: {}", e),
)
})?
.len();
// Idempotent: if blob already exists, just remove the source
if fs::try_exists(&blob_path).await.unwrap_or(false) {
let _ = fs::remove_file(&source_path).await;
return Ok(file_size);
}
// Atomic rename (same filesystem). Falls back to copy+delete for
// cross-device moves (EXDEV errno 18).
//
// Durability boundary: the caller is responsible for having
// sync_all'd the source file before invoking this function.
// (The streaming upload path writes chunks via
// `put_blob_from_bytes_unsynced` + a batched `sync_blobs`
// sweep instead; this move-based entry point remains for
// whole-file producers such as migration tooling and tests.)
// We fsync the parent of `blob_path` AFTER the rename
// so the dirent change itself becomes durable; without
// that, a power loss can resurrect the old (unrenamed)
// name even when the file contents survive.
if let Err(e) = fs::rename(&source_path, &blob_path).await {
if e.raw_os_error() == Some(18) {
// EXDEV — cross-device link. The copy() target is
// a fresh file we created, so fsync it before the
// parent-dir fsync below.
fs::copy(&source_path, &blob_path).await.map_err(|ce| {
DomainError::internal_error(
"Blob",
format!("Failed to copy file to blob store: {}", ce),
)
})?;
if let Ok(f) = fs::File::open(&blob_path).await {
let _ = f.sync_all().await;
}
let _ = fs::remove_file(&source_path).await;
} else if fs::try_exists(&blob_path).await.unwrap_or(false) {
// Concurrent writer placed the blob — discard our copy
let _ = fs::remove_file(&source_path).await;
tracing::debug!("Blob placed by concurrent writer: {}", e);
} else {
return Err(DomainError::internal_error(
"Blob",
format!("Failed to move file to blob store: {}", e),
));
}
}
fsync_parent_dir(&blob_path).await;
Ok(file_size)
})
}
fn put_blob_from_bytes(
&self,
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let blob_path = self.blob_path(&hash);
let size = data.len() as u64;
// Same durability story as `put_blob`: the blob file is
// fsync'd before the parent directory is, so both the content
// and the dirent creation survive a power loss in the same
// step. (tokio's `sync_all` flushes its internal buffer
// before issuing the fsync.)
if let Some(file) = write_blob_bytes(&blob_path, &data).await? {
file.sync_all().await.map_err(|e| {
DomainError::internal_error("Blob", format!("Failed to fsync blob file: {}", e))
})?;
drop(file);
fsync_parent_dir(&blob_path).await;
}
Ok(size)
})
}
fn put_blob_from_bytes_unsynced(
&self,
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let blob_path = self.blob_path(&hash);
let size = data.len() as u64;
if let Some(mut file) = write_blob_bytes(&blob_path, &data).await? {
// flush surfaces write errors (e.g. ENOSPC) that tokio
// would otherwise swallow on drop. It does NOT fsync —
// durability comes from the caller's later `sync_blobs`.
file.flush().await.map_err(|e| {
DomainError::internal_error("Blob", format!("Failed to flush blob file: {}", e))
})?;
}
Ok(size)
})
}
fn sync_blobs(
&self,
hashes: &[String],
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
let paths: Vec<PathBuf> = hashes.iter().map(|h| self.blob_path(h)).collect();
Box::pin(async move {
if paths.is_empty() {
return Ok(());
}
// Each distinct prefix directory is fsync'd exactly once —
// chunks of one upload land in at most 256 prefix dirs, so
// this replaces one dir fsync *per chunk* with ≤256 total.
let mut dirs: Vec<PathBuf> = paths
.iter()
.filter_map(|p| p.parent().map(Path::to_path_buf))
.collect();
dirs.sort_unstable();
dirs.dedup();
// Files first (hard requirement), then dirents (best-effort,
// same tier as fsync_parent_dir).
fsync_paths_parallel(paths, true).await?;
fsync_paths_parallel(dirs, false).await?;
Ok(())
})
}
fn get_blob_stream(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<BlobStream, DomainError>> + Send + '_>>
{
let hash = hash.to_owned();
Box::pin(async move {
let blob_path = self.blob_path(&hash);
let file = File::open(&blob_path).await.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Blob",
format!("Failed to open blob {}: {}", hash, e),
)
})?;
Ok(Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)) as BlobStream)
})
}
fn get_blob_range_stream(
&self,
hash: &str,
start: u64,
end: Option<u64>,
) -> Pin<Box<dyn std::future::Future<Output = Result<BlobStream, DomainError>> + Send + '_>>
{
let hash = hash.to_owned();
Box::pin(async move {
let blob_path = self.blob_path(&hash);
let mut file = File::open(&blob_path).await.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Blob",
format!("Failed to open blob {}: {}", hash, e),
)
})?;
file.seek(std::io::SeekFrom::Start(start))
.await
.map_err(|e| {
DomainError::internal_error("Blob", format!("Failed to seek in blob: {}", e))
})?;
if let Some(end_pos) = end {
use tokio::io::AsyncReadExt;
let limit = end_pos.saturating_sub(start);
let limited = file.take(limit);
Ok(Box::pin(ReaderStream::with_capacity(limited, STREAM_CHUNK_SIZE)) as BlobStream)
} else {
Ok(Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)) as BlobStream)
}
})
}
fn delete_blob(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let blob_path = self.blob_path(&hash);
match fs::remove_file(&blob_path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), // idempotent
Err(e) => Err(DomainError::internal_error(
"Blob",
format!("Failed to delete blob {}: {}", hash, e),
)),
}
})
}
fn blob_exists(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<bool, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let blob_path = self.blob_path(&hash);
Ok(fs::try_exists(&blob_path).await.unwrap_or(false))
})
}
fn blob_size(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let blob_path = self.blob_path(&hash);
let meta = fs::metadata(&blob_path).await.map_err(|e| {
DomainError::new(
ErrorKind::NotFound,
"Blob",
format!("Failed to stat blob {}: {}", hash, e),
)
})?;
Ok(meta.len())
})
}
fn health_check(
&self,
) -> Pin<
Box<dyn std::future::Future<Output = Result<StorageHealthStatus, DomainError>> + Send + '_>,
> {
Box::pin(async move {
let writable = fs::metadata(&self.blob_root).await.is_ok();
Ok(StorageHealthStatus {
connected: writable,
backend_type: "local".to_string(),
message: if writable {
"Local filesystem is accessible".to_string()
} else {
"Blob root directory is not accessible".to_string()
},
available_bytes: None,
})
})
}
fn backend_type(&self) -> &'static str {
"local"
}
fn local_blob_path(&self, hash: &str) -> Option<PathBuf> {
Some(self.blob_path(hash))
}
/// Local disk read-ahead for CDC reassembly. Overrides the trait default of
/// `1` with a small benchmark-backed depth (default `2`, env-tunable via
/// `OXICLOUD_LOCAL_READ_PREFETCH`). See [`DEFAULT_LOCAL_READ_PREFETCH`].
fn read_prefetch(&self) -> usize {
self.read_prefetch
}
}
#[cfg(test)]
mod tests {
use super::*;
use futures::StreamExt;
use tempfile::TempDir;
/// 64-char fake hash with the given 2-char prefix (selects the prefix dir).
fn fake_hash(prefix: &str) -> String {
format!("{prefix}{}", "0".repeat(62))
}
async fn read_blob(backend: &LocalBlobBackend, hash: &str) -> Vec<u8> {
let mut stream = backend.get_blob_stream(hash).await.unwrap();
let mut data = Vec::new();
while let Some(chunk) = stream.next().await {
data.extend_from_slice(&chunk.unwrap());
}
data
}
#[tokio::test]
async fn unsynced_write_then_sync_blobs_roundtrip() {
let tmp = TempDir::new().unwrap();
let backend = LocalBlobBackend::new(tmp.path());
backend.initialize().await.unwrap();
// Two different prefixes → exercises the distinct-parent-dir dedup.
let h1 = fake_hash("aa");
let h2 = fake_hash("bb");
backend
.put_blob_from_bytes_unsynced(&h1, Bytes::from_static(b"chunk one"))
.await
.unwrap();
backend
.put_blob_from_bytes_unsynced(&h2, Bytes::from_static(b"chunk two"))
.await
.unwrap();
backend.sync_blobs(&[h1.clone(), h2.clone()]).await.unwrap();
assert!(backend.blob_exists(&h1).await.unwrap());
assert!(backend.blob_exists(&h2).await.unwrap());
assert_eq!(read_blob(&backend, &h1).await, b"chunk one");
assert_eq!(read_blob(&backend, &h2).await, b"chunk two");
}
#[tokio::test]
async fn unsynced_write_is_idempotent() {
let tmp = TempDir::new().unwrap();
let backend = LocalBlobBackend::new(tmp.path());
backend.initialize().await.unwrap();
let hash = fake_hash("cc");
let size1 = backend
.put_blob_from_bytes_unsynced(&hash, Bytes::from_static(b"same content"))
.await
.unwrap();
let size2 = backend
.put_blob_from_bytes_unsynced(&hash, Bytes::from_static(b"same content"))
.await
.unwrap();
assert_eq!(size1, size2);
assert_eq!(read_blob(&backend, &hash).await, b"same content");
}
#[tokio::test]
async fn sync_blobs_fails_on_missing_blob() {
let tmp = TempDir::new().unwrap();
let backend = LocalBlobBackend::new(tmp.path());
backend.initialize().await.unwrap();
let missing = fake_hash("dd");
assert!(
backend.sync_blobs(&[missing]).await.is_err(),
"sweeping a never-written blob must fail — the caller would \
otherwise insert a PG row for a chunk that doesn't exist"
);
}
#[tokio::test]
async fn sync_blobs_empty_is_noop() {
let tmp = TempDir::new().unwrap();
let backend = LocalBlobBackend::new(tmp.path());
backend.initialize().await.unwrap();
backend.sync_blobs(&[]).await.unwrap();
}
}