2026-04-14 21:33:38 +02:00
|
|
|
|
//! Local Filesystem Blob Backend — stores blobs under `.blobs/{prefix}/{hash}.blob`.
|
|
|
|
|
|
//!
|
|
|
|
|
|
//! This is the default backend and a direct extraction of the filesystem I/O
|
|
|
|
|
|
//! that previously lived inside `DedupService`.
|
|
|
|
|
|
|
|
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
|
|
use std::pin::Pin;
|
|
|
|
|
|
use tokio::fs::{self, File};
|
2026-06-08 23:57:51 +02:00
|
|
|
|
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
|
2026-04-14 21:33:38 +02:00
|
|
|
|
use tokio_util::io::ReaderStream;
|
|
|
|
|
|
|
2026-04-14 23:17:39 +02:00
|
|
|
|
use bytes::Bytes;
|
|
|
|
|
|
|
2026-04-14 21:33:38 +02:00
|
|
|
|
use crate::application::ports::blob_storage_ports::{
|
|
|
|
|
|
BlobStorageBackend, BlobStream, StorageHealthStatus,
|
|
|
|
|
|
};
|
|
|
|
|
|
use crate::domain::errors::{DomainError, ErrorKind};
|
|
|
|
|
|
|
2026-06-08 23:57:51 +02:00
|
|
|
|
/// Fsync the directory containing `child_path` so a preceding rename
|
|
|
|
|
|
/// or create on `child_path` becomes durable across power loss.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// On Linux this issues `fsync(2)` on the directory file descriptor —
|
|
|
|
|
|
/// the canonical "make the dirent change durable" idiom. macOS does
|
|
|
|
|
|
/// the same but only persists to the disk controller (true persistence
|
|
|
|
|
|
/// would need `fcntl(F_FULLFSYNC)`, which tokio doesn't expose). On
|
|
|
|
|
|
/// Windows, opening a directory needs `FILE_FLAG_BACKUP_SEMANTICS` that
|
|
|
|
|
|
/// tokio's `File::open` doesn't set; that platform falls through to
|
|
|
|
|
|
/// `Ok(())` after a debug log.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Best-effort by design: a failure here is logged but does NOT fail
|
|
|
|
|
|
/// the upload, because the blob file itself was just `sync_all`'d and
|
|
|
|
|
|
/// is durable on its own. Worst case post-crash recovery: a rename
|
|
|
|
|
|
/// "reverts" to the un-renamed name (or stays renamed); the dedup-GC
|
|
|
|
|
|
/// cleanup pass handles either side.
|
|
|
|
|
|
async fn fsync_parent_dir(child_path: &Path) {
|
|
|
|
|
|
let Some(parent) = child_path.parent() else {
|
|
|
|
|
|
return;
|
|
|
|
|
|
};
|
|
|
|
|
|
let parent = parent.to_owned();
|
|
|
|
|
|
// std::fs (synchronous) opens directories reliably on Linux/macOS;
|
|
|
|
|
|
// do it on the blocking pool so we don't park the tokio worker.
|
|
|
|
|
|
let result = tokio::task::spawn_blocking(move || -> std::io::Result<()> {
|
|
|
|
|
|
let dir = std::fs::File::open(&parent)?;
|
|
|
|
|
|
dir.sync_all()
|
|
|
|
|
|
})
|
|
|
|
|
|
.await;
|
|
|
|
|
|
match result {
|
|
|
|
|
|
Ok(Ok(())) => {}
|
|
|
|
|
|
Ok(Err(e)) => {
|
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
error = %e,
|
|
|
|
|
|
path = %child_path.display(),
|
|
|
|
|
|
"Blob parent-dir fsync failed (rename durability not guaranteed)"
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
error = %e,
|
|
|
|
|
|
path = %child_path.display(),
|
|
|
|
|
|
"Blob parent-dir fsync task join failed"
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-14 21:33:38 +02:00
|
|
|
|
/// Chunk size for streaming file reads (256 KB).
|
|
|
|
|
|
const STREAM_CHUNK_SIZE: usize = 256 * 1024;
|
|
|
|
|
|
|
2026-06-10 09:55:02 +00:00
|
|
|
|
/// Max parallel blocking tasks for the [`fsync_paths_parallel`] sweep.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Concurrent fsyncs let journaling filesystems coalesce barriers (ext4
|
|
|
|
|
|
/// merges parallel fsyncs into shared journal commits), so a sweep over
|
|
|
|
|
|
/// thousands of chunk files costs a small fraction of issuing the same
|
|
|
|
|
|
/// fsyncs sequentially.
|
|
|
|
|
|
const SYNC_SWEEP_CONCURRENCY: usize = 16;
|
|
|
|
|
|
|
|
|
|
|
|
/// Fsync every path in `paths`, spread over up to
|
|
|
|
|
|
/// [`SYNC_SWEEP_CONCURRENCY`] blocking-pool tasks.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// `strict` mirrors the two durability tiers already present in this
|
|
|
|
|
|
/// module: blob *files* must be durable (hard error on failure, like
|
|
|
|
|
|
/// `put_blob_from_bytes`), while *directory* fsyncs are best-effort
|
|
|
|
|
|
/// (logged warning, like [`fsync_parent_dir`]) — directories can't be
|
|
|
|
|
|
/// opened for fsync on every platform.
|
|
|
|
|
|
async fn fsync_paths_parallel(paths: Vec<PathBuf>, strict: bool) -> Result<(), DomainError> {
|
|
|
|
|
|
if paths.is_empty() {
|
|
|
|
|
|
return Ok(());
|
|
|
|
|
|
}
|
|
|
|
|
|
let group_size = paths.len().div_ceil(SYNC_SWEEP_CONCURRENCY);
|
|
|
|
|
|
let mut tasks = Vec::with_capacity(SYNC_SWEEP_CONCURRENCY);
|
|
|
|
|
|
for group in paths.chunks(group_size) {
|
|
|
|
|
|
let group = group.to_vec();
|
|
|
|
|
|
tasks.push(tokio::task::spawn_blocking(
|
|
|
|
|
|
move || -> Result<(), (PathBuf, std::io::Error)> {
|
|
|
|
|
|
for path in &group {
|
|
|
|
|
|
let result = std::fs::File::open(path).and_then(|f| f.sync_all());
|
|
|
|
|
|
if let Err(e) = result {
|
|
|
|
|
|
if strict {
|
|
|
|
|
|
return Err((path.clone(), e));
|
|
|
|
|
|
}
|
|
|
|
|
|
tracing::warn!(
|
|
|
|
|
|
error = %e,
|
|
|
|
|
|
path = %path.display(),
|
|
|
|
|
|
"Blob sync sweep: best-effort fsync failed"
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
},
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
for task in tasks {
|
|
|
|
|
|
task.await
|
|
|
|
|
|
.map_err(|e| DomainError::internal_error("Blob", format!("sync sweep join: {e}")))?
|
|
|
|
|
|
.map_err(|(path, e)| {
|
|
|
|
|
|
DomainError::internal_error(
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("sync sweep fsync of {} failed: {e}", path.display()),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Create `blob_path` and write `data` into it.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Returns the open file handle so the caller decides the durability tier
|
|
|
|
|
|
/// (fsync now vs. deferred batch sync), or `None` when the blob already
|
|
|
|
|
|
/// existed (idempotent skip — content-addressed, so identical by definition).
|
|
|
|
|
|
async fn write_blob_bytes(blob_path: &Path, data: &Bytes) -> Result<Option<File>, DomainError> {
|
2026-07-18 16:12:04 +00:00
|
|
|
|
// One atomic O_CREAT|O_EXCL open replaces the old stat-then-create pair:
|
|
|
|
|
|
// `AlreadyExists` IS the idempotent skip (content-addressed names mean an
|
|
|
|
|
|
// existing file has identical content), saving a syscall + a blocking-pool
|
|
|
|
|
|
// dispatch on every new chunk of every upload.
|
|
|
|
|
|
let mut file = match fs::File::options()
|
|
|
|
|
|
.write(true)
|
|
|
|
|
|
.create_new(true)
|
|
|
|
|
|
.open(blob_path)
|
|
|
|
|
|
.await
|
|
|
|
|
|
{
|
|
|
|
|
|
Ok(f) => f,
|
|
|
|
|
|
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => return Ok(None),
|
|
|
|
|
|
Err(e) => {
|
|
|
|
|
|
return Err(DomainError::internal_error(
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("Failed to create blob file: {}", e),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
};
|
2026-06-10 09:55:02 +00:00
|
|
|
|
file.write_all(data).await.map_err(|e| {
|
|
|
|
|
|
DomainError::internal_error("Blob", format!("Failed to write blob from bytes: {}", e))
|
|
|
|
|
|
})?;
|
|
|
|
|
|
Ok(Some(file))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-07-18 16:12:04 +00:00
|
|
|
|
/// Bench-only public wrapper (feature = "bench") over the private chunk
|
|
|
|
|
|
/// writer so `examples/bench_storage_micro.rs` can A/B the open strategy.
|
|
|
|
|
|
#[cfg(feature = "bench")]
|
|
|
|
|
|
pub async fn write_blob_bytes_for_bench(
|
|
|
|
|
|
blob_path: &Path,
|
|
|
|
|
|
data: &Bytes,
|
|
|
|
|
|
) -> Result<Option<File>, DomainError> {
|
|
|
|
|
|
write_blob_bytes(blob_path, data).await
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-14 21:33:38 +02:00
|
|
|
|
/// Compile-time lookup table for the 256 two-digit lowercase hex prefixes ("00"…"ff").
|
|
|
|
|
|
static HEX_PREFIXES: [&str; 256] = [
|
|
|
|
|
|
"00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0a", "0b", "0c", "0d", "0e", "0f",
|
|
|
|
|
|
"10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "1a", "1b", "1c", "1d", "1e", "1f",
|
|
|
|
|
|
"20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "2a", "2b", "2c", "2d", "2e", "2f",
|
|
|
|
|
|
"30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "3a", "3b", "3c", "3d", "3e", "3f",
|
|
|
|
|
|
"40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "4a", "4b", "4c", "4d", "4e", "4f",
|
|
|
|
|
|
"50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "5a", "5b", "5c", "5d", "5e", "5f",
|
|
|
|
|
|
"60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "6a", "6b", "6c", "6d", "6e", "6f",
|
|
|
|
|
|
"70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "7a", "7b", "7c", "7d", "7e", "7f",
|
|
|
|
|
|
"80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "8a", "8b", "8c", "8d", "8e", "8f",
|
|
|
|
|
|
"90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "9a", "9b", "9c", "9d", "9e", "9f",
|
|
|
|
|
|
"a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9", "aa", "ab", "ac", "ad", "ae", "af",
|
|
|
|
|
|
"b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9", "ba", "bb", "bc", "bd", "be", "bf",
|
|
|
|
|
|
"c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "ca", "cb", "cc", "cd", "ce", "cf",
|
|
|
|
|
|
"d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "da", "db", "dc", "dd", "de", "df",
|
|
|
|
|
|
"e0", "e1", "e2", "e3", "e4", "e5", "e6", "e7", "e8", "e9", "ea", "eb", "ec", "ed", "ee", "ef",
|
|
|
|
|
|
"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "fa", "fb", "fc", "fd", "fe", "ff",
|
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
|
|
/// Local filesystem blob backend.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Blobs are stored under `blob_root/{2-char-prefix}/{hash}.blob`.
|
|
|
|
|
|
/// Temporary upload staging uses `temp_root/`.
|
|
|
|
|
|
pub struct LocalBlobBackend {
|
|
|
|
|
|
blob_root: PathBuf,
|
|
|
|
|
|
temp_root: PathBuf,
|
2026-06-22 08:10:46 +00:00
|
|
|
|
/// Chunk read-ahead depth for CDC reassembly — see [`Self::new`].
|
|
|
|
|
|
read_prefetch: usize,
|
2026-04-14 21:33:38 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-22 08:10:46 +00:00
|
|
|
|
/// Default chunk-open read-ahead for the local backend (overrides the trait's
|
|
|
|
|
|
/// conservative `1`).
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Benchmarked with `examples/bench_blob_prefetch` on SSD-class storage: a small
|
|
|
|
|
|
/// read-ahead is the sweet spot for the *disk-bound* read paths — localhost/LAN
|
|
|
|
|
|
/// downloads and, importantly, the internal blob reads that drain as fast as the
|
|
|
|
|
|
/// disk delivers (thumbnail render, transcode, ZIP export, content extraction),
|
|
|
|
|
|
/// all of which flow through `DedupService::stream_chunks`'s `buffered(N)`.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Measured median throughput vs the old sequential `N=1`:
|
|
|
|
|
|
/// warm disk-bound +11.8% (N=2) cold disk-bound +7.2% (N=2)
|
|
|
|
|
|
/// network-bound (throttled) ≈ 0% — the consumer, not the disk, is the cap
|
|
|
|
|
|
/// N=16 −4.4% warm — fan-out past a couple turns one sequential read into
|
|
|
|
|
|
/// competing random I/O over scattered content-addressed chunk files.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// `2` deliberately captures most of that gain at the lowest fan-out, because
|
|
|
|
|
|
/// `buffered(N)` here overlaps the per-chunk `File::open` (cheap on local disk),
|
|
|
|
|
|
/// not the data read, so deeper queues buy little and risk seek contention on
|
|
|
|
|
|
/// the spinning disks we can't bench here. Operators tune it via
|
|
|
|
|
|
/// `OXICLOUD_LOCAL_READ_PREFETCH` (set `1` on seek-bound HDDs to restore the old
|
|
|
|
|
|
/// strictly-sequential behaviour; raise it on fast NVMe arrays).
|
|
|
|
|
|
const DEFAULT_LOCAL_READ_PREFETCH: usize = 2;
|
|
|
|
|
|
|
2026-04-14 21:33:38 +02:00
|
|
|
|
impl LocalBlobBackend {
|
|
|
|
|
|
/// Create a new local backend rooted at `storage_root`.
|
|
|
|
|
|
///
|
|
|
|
|
|
/// Blob files go under `{storage_root}/.blobs/`, temp files under
|
|
|
|
|
|
/// `{storage_root}/.dedup_temp/`.
|
|
|
|
|
|
pub fn new(storage_root: &Path) -> Self {
|
2026-06-22 08:10:46 +00:00
|
|
|
|
// Read-ahead depth: env override, else the benchmark-backed default.
|
|
|
|
|
|
// Clamped to ≥1 so a bogus `0` can't stall reads (buffered(0) would
|
|
|
|
|
|
// make no progress; `stream_chunks` also guards with `.max(1)`).
|
|
|
|
|
|
let read_prefetch = std::env::var("OXICLOUD_LOCAL_READ_PREFETCH")
|
|
|
|
|
|
.ok()
|
|
|
|
|
|
.and_then(|v| v.parse::<usize>().ok())
|
|
|
|
|
|
.map(|n| n.max(1))
|
|
|
|
|
|
.unwrap_or(DEFAULT_LOCAL_READ_PREFETCH);
|
2026-04-14 21:33:38 +02:00
|
|
|
|
Self {
|
|
|
|
|
|
blob_root: storage_root.join(".blobs"),
|
|
|
|
|
|
temp_root: storage_root.join(".dedup_temp"),
|
2026-06-22 08:10:46 +00:00
|
|
|
|
read_prefetch,
|
2026-04-14 21:33:38 +02:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Compute the filesystem path for a blob hash.
|
|
|
|
|
|
pub fn blob_path(&self, hash: &str) -> PathBuf {
|
|
|
|
|
|
let prefix = &hash[0..2];
|
|
|
|
|
|
self.blob_root.join(prefix).join(format!("{}.blob", hash))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// Return a reference to the blob root directory.
|
|
|
|
|
|
pub fn blob_root(&self) -> &Path {
|
|
|
|
|
|
&self.blob_root
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl BlobStorageBackend for LocalBlobBackend {
|
|
|
|
|
|
fn initialize(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
fs::create_dir_all(&self.blob_root)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(DomainError::from)?;
|
|
|
|
|
|
fs::create_dir_all(&self.temp_root)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(DomainError::from)?;
|
|
|
|
|
|
|
|
|
|
|
|
// Create the 256 hash-prefix directories (00-ff)
|
|
|
|
|
|
for prefix in &HEX_PREFIXES {
|
|
|
|
|
|
fs::create_dir_all(self.blob_root.join(prefix))
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(DomainError::from)?;
|
|
|
|
|
|
}
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn put_blob(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hash: &str,
|
|
|
|
|
|
source_path: &Path,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
|
|
|
|
|
let hash = hash.to_owned();
|
|
|
|
|
|
let source_path = source_path.to_owned();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let blob_path = self.blob_path(&hash);
|
|
|
|
|
|
|
|
|
|
|
|
let file_size = fs::metadata(&source_path)
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| {
|
|
|
|
|
|
DomainError::internal_error(
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("Failed to stat source file: {}", e),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?
|
|
|
|
|
|
.len();
|
|
|
|
|
|
|
|
|
|
|
|
// Idempotent: if blob already exists, just remove the source
|
|
|
|
|
|
if fs::try_exists(&blob_path).await.unwrap_or(false) {
|
|
|
|
|
|
let _ = fs::remove_file(&source_path).await;
|
|
|
|
|
|
return Ok(file_size);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Atomic rename (same filesystem). Falls back to copy+delete for
|
|
|
|
|
|
// cross-device moves (EXDEV errno 18).
|
2026-06-08 23:57:51 +02:00
|
|
|
|
//
|
2026-06-11 13:06:33 +00:00
|
|
|
|
// Durability boundary: the caller is responsible for having
|
|
|
|
|
|
// sync_all'd the source file before invoking this function.
|
|
|
|
|
|
// (The streaming upload path writes chunks via
|
|
|
|
|
|
// `put_blob_from_bytes_unsynced` + a batched `sync_blobs`
|
|
|
|
|
|
// sweep instead; this move-based entry point remains for
|
|
|
|
|
|
// whole-file producers such as migration tooling and tests.)
|
|
|
|
|
|
// We fsync the parent of `blob_path` AFTER the rename
|
2026-06-08 23:57:51 +02:00
|
|
|
|
// so the dirent change itself becomes durable; without
|
|
|
|
|
|
// that, a power loss can resurrect the old (unrenamed)
|
|
|
|
|
|
// name even when the file contents survive.
|
2026-04-14 21:33:38 +02:00
|
|
|
|
if let Err(e) = fs::rename(&source_path, &blob_path).await {
|
|
|
|
|
|
if e.raw_os_error() == Some(18) {
|
2026-06-08 23:57:51 +02:00
|
|
|
|
// EXDEV — cross-device link. The copy() target is
|
|
|
|
|
|
// a fresh file we created, so fsync it before the
|
|
|
|
|
|
// parent-dir fsync below.
|
2026-04-14 21:33:38 +02:00
|
|
|
|
fs::copy(&source_path, &blob_path).await.map_err(|ce| {
|
|
|
|
|
|
DomainError::internal_error(
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("Failed to copy file to blob store: {}", ce),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
2026-06-08 23:57:51 +02:00
|
|
|
|
if let Ok(f) = fs::File::open(&blob_path).await {
|
|
|
|
|
|
let _ = f.sync_all().await;
|
|
|
|
|
|
}
|
2026-04-14 21:33:38 +02:00
|
|
|
|
let _ = fs::remove_file(&source_path).await;
|
|
|
|
|
|
} else if fs::try_exists(&blob_path).await.unwrap_or(false) {
|
|
|
|
|
|
// Concurrent writer placed the blob — discard our copy
|
|
|
|
|
|
let _ = fs::remove_file(&source_path).await;
|
|
|
|
|
|
tracing::debug!("Blob placed by concurrent writer: {}", e);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
return Err(DomainError::internal_error(
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("Failed to move file to blob store: {}", e),
|
|
|
|
|
|
));
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-08 23:57:51 +02:00
|
|
|
|
fsync_parent_dir(&blob_path).await;
|
|
|
|
|
|
|
2026-04-14 21:33:38 +02:00
|
|
|
|
Ok(file_size)
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-14 23:17:39 +02:00
|
|
|
|
fn put_blob_from_bytes(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hash: &str,
|
|
|
|
|
|
data: Bytes,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
|
|
|
|
|
let hash = hash.to_owned();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let blob_path = self.blob_path(&hash);
|
|
|
|
|
|
let size = data.len() as u64;
|
|
|
|
|
|
|
2026-06-10 09:55:02 +00:00
|
|
|
|
// Same durability story as `put_blob`: the blob file is
|
|
|
|
|
|
// fsync'd before the parent directory is, so both the content
|
|
|
|
|
|
// and the dirent creation survive a power loss in the same
|
|
|
|
|
|
// step. (tokio's `sync_all` flushes its internal buffer
|
|
|
|
|
|
// before issuing the fsync.)
|
|
|
|
|
|
if let Some(file) = write_blob_bytes(&blob_path, &data).await? {
|
|
|
|
|
|
file.sync_all().await.map_err(|e| {
|
|
|
|
|
|
DomainError::internal_error("Blob", format!("Failed to fsync blob file: {}", e))
|
|
|
|
|
|
})?;
|
|
|
|
|
|
drop(file);
|
|
|
|
|
|
fsync_parent_dir(&blob_path).await;
|
2026-04-14 23:17:39 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-10 09:55:02 +00:00
|
|
|
|
Ok(size)
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn put_blob_from_bytes_unsynced(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hash: &str,
|
|
|
|
|
|
data: Bytes,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
|
|
|
|
|
let hash = hash.to_owned();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let blob_path = self.blob_path(&hash);
|
|
|
|
|
|
let size = data.len() as u64;
|
|
|
|
|
|
|
|
|
|
|
|
if let Some(mut file) = write_blob_bytes(&blob_path, &data).await? {
|
|
|
|
|
|
// flush surfaces write errors (e.g. ENOSPC) that tokio
|
|
|
|
|
|
// would otherwise swallow on drop. It does NOT fsync —
|
|
|
|
|
|
// durability comes from the caller's later `sync_blobs`.
|
|
|
|
|
|
file.flush().await.map_err(|e| {
|
|
|
|
|
|
DomainError::internal_error("Blob", format!("Failed to flush blob file: {}", e))
|
|
|
|
|
|
})?;
|
|
|
|
|
|
}
|
2026-04-14 23:17:39 +02:00
|
|
|
|
|
|
|
|
|
|
Ok(size)
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-06-10 09:55:02 +00:00
|
|
|
|
fn sync_blobs(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hashes: &[String],
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
|
|
|
|
|
|
let paths: Vec<PathBuf> = hashes.iter().map(|h| self.blob_path(h)).collect();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
if paths.is_empty() {
|
|
|
|
|
|
return Ok(());
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Each distinct prefix directory is fsync'd exactly once —
|
|
|
|
|
|
// chunks of one upload land in at most 256 prefix dirs, so
|
|
|
|
|
|
// this replaces one dir fsync *per chunk* with ≤256 total.
|
|
|
|
|
|
let mut dirs: Vec<PathBuf> = paths
|
|
|
|
|
|
.iter()
|
|
|
|
|
|
.filter_map(|p| p.parent().map(Path::to_path_buf))
|
|
|
|
|
|
.collect();
|
|
|
|
|
|
dirs.sort_unstable();
|
|
|
|
|
|
dirs.dedup();
|
|
|
|
|
|
|
|
|
|
|
|
// Files first (hard requirement), then dirents (best-effort,
|
|
|
|
|
|
// same tier as fsync_parent_dir).
|
|
|
|
|
|
fsync_paths_parallel(paths, true).await?;
|
|
|
|
|
|
fsync_paths_parallel(dirs, false).await?;
|
|
|
|
|
|
Ok(())
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2026-04-14 21:33:38 +02:00
|
|
|
|
fn get_blob_stream(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hash: &str,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<BlobStream, DomainError>> + Send + '_>>
|
|
|
|
|
|
{
|
|
|
|
|
|
let hash = hash.to_owned();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let blob_path = self.blob_path(&hash);
|
|
|
|
|
|
let file = File::open(&blob_path).await.map_err(|e| {
|
|
|
|
|
|
DomainError::new(
|
|
|
|
|
|
ErrorKind::NotFound,
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("Failed to open blob {}: {}", hash, e),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
Ok(Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)) as BlobStream)
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn get_blob_range_stream(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hash: &str,
|
|
|
|
|
|
start: u64,
|
|
|
|
|
|
end: Option<u64>,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<BlobStream, DomainError>> + Send + '_>>
|
|
|
|
|
|
{
|
|
|
|
|
|
let hash = hash.to_owned();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let blob_path = self.blob_path(&hash);
|
|
|
|
|
|
let mut file = File::open(&blob_path).await.map_err(|e| {
|
|
|
|
|
|
DomainError::new(
|
|
|
|
|
|
ErrorKind::NotFound,
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("Failed to open blob {}: {}", hash, e),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
|
|
|
|
|
|
file.seek(std::io::SeekFrom::Start(start))
|
|
|
|
|
|
.await
|
|
|
|
|
|
.map_err(|e| {
|
|
|
|
|
|
DomainError::internal_error("Blob", format!("Failed to seek in blob: {}", e))
|
|
|
|
|
|
})?;
|
|
|
|
|
|
|
|
|
|
|
|
if let Some(end_pos) = end {
|
|
|
|
|
|
use tokio::io::AsyncReadExt;
|
|
|
|
|
|
let limit = end_pos.saturating_sub(start);
|
|
|
|
|
|
let limited = file.take(limit);
|
|
|
|
|
|
Ok(Box::pin(ReaderStream::with_capacity(limited, STREAM_CHUNK_SIZE)) as BlobStream)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
Ok(Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)) as BlobStream)
|
|
|
|
|
|
}
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn delete_blob(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hash: &str,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
|
|
|
|
|
|
let hash = hash.to_owned();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let blob_path = self.blob_path(&hash);
|
|
|
|
|
|
match fs::remove_file(&blob_path).await {
|
|
|
|
|
|
Ok(()) => Ok(()),
|
|
|
|
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), // idempotent
|
|
|
|
|
|
Err(e) => Err(DomainError::internal_error(
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("Failed to delete blob {}: {}", hash, e),
|
|
|
|
|
|
)),
|
|
|
|
|
|
}
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn blob_exists(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hash: &str,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<bool, DomainError>> + Send + '_>> {
|
|
|
|
|
|
let hash = hash.to_owned();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let blob_path = self.blob_path(&hash);
|
|
|
|
|
|
Ok(fs::try_exists(&blob_path).await.unwrap_or(false))
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn blob_size(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
hash: &str,
|
|
|
|
|
|
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
|
|
|
|
|
|
let hash = hash.to_owned();
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let blob_path = self.blob_path(&hash);
|
|
|
|
|
|
let meta = fs::metadata(&blob_path).await.map_err(|e| {
|
|
|
|
|
|
DomainError::new(
|
|
|
|
|
|
ErrorKind::NotFound,
|
|
|
|
|
|
"Blob",
|
|
|
|
|
|
format!("Failed to stat blob {}: {}", hash, e),
|
|
|
|
|
|
)
|
|
|
|
|
|
})?;
|
|
|
|
|
|
Ok(meta.len())
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn health_check(
|
|
|
|
|
|
&self,
|
|
|
|
|
|
) -> Pin<
|
|
|
|
|
|
Box<dyn std::future::Future<Output = Result<StorageHealthStatus, DomainError>> + Send + '_>,
|
|
|
|
|
|
> {
|
|
|
|
|
|
Box::pin(async move {
|
|
|
|
|
|
let writable = fs::metadata(&self.blob_root).await.is_ok();
|
|
|
|
|
|
Ok(StorageHealthStatus {
|
|
|
|
|
|
connected: writable,
|
|
|
|
|
|
backend_type: "local".to_string(),
|
|
|
|
|
|
message: if writable {
|
|
|
|
|
|
"Local filesystem is accessible".to_string()
|
|
|
|
|
|
} else {
|
|
|
|
|
|
"Blob root directory is not accessible".to_string()
|
|
|
|
|
|
},
|
|
|
|
|
|
available_bytes: None,
|
|
|
|
|
|
})
|
|
|
|
|
|
})
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn backend_type(&self) -> &'static str {
|
|
|
|
|
|
"local"
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn local_blob_path(&self, hash: &str) -> Option<PathBuf> {
|
|
|
|
|
|
Some(self.blob_path(hash))
|
|
|
|
|
|
}
|
2026-06-22 08:10:46 +00:00
|
|
|
|
|
|
|
|
|
|
/// Local disk read-ahead for CDC reassembly. Overrides the trait default of
|
|
|
|
|
|
/// `1` with a small benchmark-backed depth (default `2`, env-tunable via
|
|
|
|
|
|
/// `OXICLOUD_LOCAL_READ_PREFETCH`). See [`DEFAULT_LOCAL_READ_PREFETCH`].
|
|
|
|
|
|
fn read_prefetch(&self) -> usize {
|
|
|
|
|
|
self.read_prefetch
|
|
|
|
|
|
}
|
2026-04-14 21:33:38 +02:00
|
|
|
|
}
|
2026-06-10 09:55:02 +00:00
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
|
mod tests {
|
|
|
|
|
|
use super::*;
|
|
|
|
|
|
use futures::StreamExt;
|
|
|
|
|
|
use tempfile::TempDir;
|
|
|
|
|
|
|
|
|
|
|
|
/// 64-char fake hash with the given 2-char prefix (selects the prefix dir).
|
|
|
|
|
|
fn fake_hash(prefix: &str) -> String {
|
|
|
|
|
|
format!("{prefix}{}", "0".repeat(62))
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
async fn read_blob(backend: &LocalBlobBackend, hash: &str) -> Vec<u8> {
|
|
|
|
|
|
let mut stream = backend.get_blob_stream(hash).await.unwrap();
|
|
|
|
|
|
let mut data = Vec::new();
|
|
|
|
|
|
while let Some(chunk) = stream.next().await {
|
|
|
|
|
|
data.extend_from_slice(&chunk.unwrap());
|
|
|
|
|
|
}
|
|
|
|
|
|
data
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn unsynced_write_then_sync_blobs_roundtrip() {
|
|
|
|
|
|
let tmp = TempDir::new().unwrap();
|
|
|
|
|
|
let backend = LocalBlobBackend::new(tmp.path());
|
|
|
|
|
|
backend.initialize().await.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
// Two different prefixes → exercises the distinct-parent-dir dedup.
|
|
|
|
|
|
let h1 = fake_hash("aa");
|
|
|
|
|
|
let h2 = fake_hash("bb");
|
|
|
|
|
|
backend
|
|
|
|
|
|
.put_blob_from_bytes_unsynced(&h1, Bytes::from_static(b"chunk one"))
|
|
|
|
|
|
.await
|
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
backend
|
|
|
|
|
|
.put_blob_from_bytes_unsynced(&h2, Bytes::from_static(b"chunk two"))
|
|
|
|
|
|
.await
|
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
backend.sync_blobs(&[h1.clone(), h2.clone()]).await.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
assert!(backend.blob_exists(&h1).await.unwrap());
|
|
|
|
|
|
assert!(backend.blob_exists(&h2).await.unwrap());
|
|
|
|
|
|
assert_eq!(read_blob(&backend, &h1).await, b"chunk one");
|
|
|
|
|
|
assert_eq!(read_blob(&backend, &h2).await, b"chunk two");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn unsynced_write_is_idempotent() {
|
|
|
|
|
|
let tmp = TempDir::new().unwrap();
|
|
|
|
|
|
let backend = LocalBlobBackend::new(tmp.path());
|
|
|
|
|
|
backend.initialize().await.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
let hash = fake_hash("cc");
|
|
|
|
|
|
let size1 = backend
|
|
|
|
|
|
.put_blob_from_bytes_unsynced(&hash, Bytes::from_static(b"same content"))
|
|
|
|
|
|
.await
|
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
let size2 = backend
|
|
|
|
|
|
.put_blob_from_bytes_unsynced(&hash, Bytes::from_static(b"same content"))
|
|
|
|
|
|
.await
|
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
assert_eq!(size1, size2);
|
|
|
|
|
|
assert_eq!(read_blob(&backend, &hash).await, b"same content");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn sync_blobs_fails_on_missing_blob() {
|
|
|
|
|
|
let tmp = TempDir::new().unwrap();
|
|
|
|
|
|
let backend = LocalBlobBackend::new(tmp.path());
|
|
|
|
|
|
backend.initialize().await.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
let missing = fake_hash("dd");
|
|
|
|
|
|
assert!(
|
|
|
|
|
|
backend.sync_blobs(&[missing]).await.is_err(),
|
|
|
|
|
|
"sweeping a never-written blob must fail — the caller would \
|
|
|
|
|
|
otherwise insert a PG row for a chunk that doesn't exist"
|
|
|
|
|
|
);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
|
async fn sync_blobs_empty_is_noop() {
|
|
|
|
|
|
let tmp = TempDir::new().unwrap();
|
|
|
|
|
|
let backend = LocalBlobBackend::new(tmp.path());
|
|
|
|
|
|
backend.initialize().await.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
backend.sync_blobs(&[]).await.unwrap();
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|