perf(blobs): batch chunk fsyncs into one durability sweep per upload

Storing a new file through CDC dedup issued sync_all + a parent-dir
fsync for every ~256 KB chunk (~8,200 fsyncs for a 1 GB upload), plus
one PG INSERT round-trip per chunk. The actual durability boundary is
the manifest INSERT: chunks only need to be durable before any PG row
references them, not one by one.

- BlobStorageBackend grows put_blob_from_bytes_unsynced + sync_blobs
  with conservative defaults (unsynced delegates to the synced write,
  sync_blobs is a no-op) so backends that don't opt in keep the
  per-write durability semantics. Remote stores are durable on PUT.
- LocalBlobBackend writes chunks without fsync and implements
  sync_blobs as a parallel sweep: every listed blob file (hard
  requirement) plus each distinct prefix directory exactly once
  (best-effort, same tier as fsync_parent_dir).
- DedupService::store_chunks writes new chunks unsynced, runs one
  sync_blobs sweep, then registers all new chunks in ONE batched
  UNNEST INSERT - durability before visibility, and the per-chunk PG
  round-trips collapse into one.
- Encrypted/Migration decorators forward both methods so the
  optimization survives encrypted-local and live-migration stacks.

https://claude.ai/code/session_013Bk4BMQEvR9QxCU7QXLRwv

src/infrastructure/services/local_blob_backend.rs

@@ -66,6 +66,80 @@ async fn fsync_parent_dir(child_path: &Path) {
 /// Chunk size for streaming file reads (256 KB).
 const STREAM_CHUNK_SIZE: usize = 256 * 1024;
 
+/// Max parallel blocking tasks for the [`fsync_paths_parallel`] sweep.
+///
+/// Concurrent fsyncs let journaling filesystems coalesce barriers (ext4
+/// merges parallel fsyncs into shared journal commits), so a sweep over
+/// thousands of chunk files costs a small fraction of issuing the same
+/// fsyncs sequentially.
+const SYNC_SWEEP_CONCURRENCY: usize = 16;
+
+/// Fsync every path in `paths`, spread over up to
+/// [`SYNC_SWEEP_CONCURRENCY`] blocking-pool tasks.
+///
+/// `strict` mirrors the two durability tiers already present in this
+/// module: blob *files* must be durable (hard error on failure, like
+/// `put_blob_from_bytes`), while *directory* fsyncs are best-effort
+/// (logged warning, like [`fsync_parent_dir`]) — directories can't be
+/// opened for fsync on every platform.
+async fn fsync_paths_parallel(paths: Vec<PathBuf>, strict: bool) -> Result<(), DomainError> {
+    if paths.is_empty() {
+        return Ok(());
+    }
+    let group_size = paths.len().div_ceil(SYNC_SWEEP_CONCURRENCY);
+    let mut tasks = Vec::with_capacity(SYNC_SWEEP_CONCURRENCY);
+    for group in paths.chunks(group_size) {
+        let group = group.to_vec();
+        tasks.push(tokio::task::spawn_blocking(
+            move || -> Result<(), (PathBuf, std::io::Error)> {
+                for path in &group {
+                    let result = std::fs::File::open(path).and_then(|f| f.sync_all());
+                    if let Err(e) = result {
+                        if strict {
+                            return Err((path.clone(), e));
+                        }
+                        tracing::warn!(
+                            error = %e,
+                            path = %path.display(),
+                            "Blob sync sweep: best-effort fsync failed"
+                        );
+                    }
+                }
+                Ok(())
+            },
+        ));
+    }
+    for task in tasks {
+        task.await
+            .map_err(|e| DomainError::internal_error("Blob", format!("sync sweep join: {e}")))?
+            .map_err(|(path, e)| {
+                DomainError::internal_error(
+                    "Blob",
+                    format!("sync sweep fsync of {} failed: {e}", path.display()),
+                )
+            })?;
+    }
+    Ok(())
+}
+
+/// Create `blob_path` and write `data` into it.
+///
+/// Returns the open file handle so the caller decides the durability tier
+/// (fsync now vs. deferred batch sync), or `None` when the blob already
+/// existed (idempotent skip — content-addressed, so identical by definition).
+async fn write_blob_bytes(blob_path: &Path, data: &Bytes) -> Result<Option<File>, DomainError> {
+    if fs::try_exists(blob_path).await.unwrap_or(false) {
+        return Ok(None);
+    }
+    let mut file = fs::File::create(blob_path).await.map_err(|e| {
+        DomainError::internal_error("Blob", format!("Failed to create blob file: {}", e))
+    })?;
+    file.write_all(data).await.map_err(|e| {
+        DomainError::internal_error("Blob", format!("Failed to write blob from bytes: {}", e))
+    })?;
+    Ok(Some(file))
+}
+
 /// Compile-time lookup table for the 256 two-digit lowercase hex prefixes ("00"…"ff").
 static HEX_PREFIXES: [&str; 256] = [
     "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0a", "0b", "0c", "0d", "0e", "0f",
@@ -222,37 +296,74 @@ impl BlobStorageBackend for LocalBlobBackend {
             let blob_path = self.blob_path(&hash);
             let size = data.len() as u64;
 
-            // Idempotent: if blob already exists, skip
-            if fs::try_exists(&blob_path).await.unwrap_or(false) {
-                return Ok(size);
+            // Same durability story as `put_blob`: the blob file is
+            // fsync'd before the parent directory is, so both the content
+            // and the dirent creation survive a power loss in the same
+            // step. (tokio's `sync_all` flushes its internal buffer
+            // before issuing the fsync.)
+            if let Some(file) = write_blob_bytes(&blob_path, &data).await? {
+                file.sync_all().await.map_err(|e| {
+                    DomainError::internal_error("Blob", format!("Failed to fsync blob file: {}", e))
+                })?;
+                drop(file);
+                fsync_parent_dir(&blob_path).await;
             }
 
-            // Write directly to blob path. `fs::write` is `create +
-            // write_all + close` — but the close on tokio::fs::File
-            // does NOT fsync, so we open explicitly to keep the
-            // `sync_all` call site obvious. Same durability story as
-            // `put_blob`: the blob file is fsync'd before the parent
-            // directory is, so both the content and the dirent
-            // creation survive a power loss in the same step.
-            let mut file = fs::File::create(&blob_path).await.map_err(|e| {
-                DomainError::internal_error("Blob", format!("Failed to create blob file: {}", e))
-            })?;
-            file.write_all(&data).await.map_err(|e| {
-                DomainError::internal_error(
-                    "Blob",
-                    format!("Failed to write blob from bytes: {}", e),
-                )
-            })?;
-            file.sync_all().await.map_err(|e| {
-                DomainError::internal_error("Blob", format!("Failed to fsync blob file: {}", e))
-            })?;
-            drop(file);
-            fsync_parent_dir(&blob_path).await;
+            Ok(size)
+        })
+    }
+
+    fn put_blob_from_bytes_unsynced(
+        &self,
+        hash: &str,
+        data: Bytes,
+    ) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
+        let hash = hash.to_owned();
+        Box::pin(async move {
+            let blob_path = self.blob_path(&hash);
+            let size = data.len() as u64;
+
+            if let Some(mut file) = write_blob_bytes(&blob_path, &data).await? {
+                // flush surfaces write errors (e.g. ENOSPC) that tokio
+                // would otherwise swallow on drop. It does NOT fsync —
+                // durability comes from the caller's later `sync_blobs`.
+                file.flush().await.map_err(|e| {
+                    DomainError::internal_error("Blob", format!("Failed to flush blob file: {}", e))
+                })?;
+            }
 
             Ok(size)
         })
     }
 
+    fn sync_blobs(
+        &self,
+        hashes: &[String],
+    ) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
+        let paths: Vec<PathBuf> = hashes.iter().map(|h| self.blob_path(h)).collect();
+        Box::pin(async move {
+            if paths.is_empty() {
+                return Ok(());
+            }
+
+            // Each distinct prefix directory is fsync'd exactly once —
+            // chunks of one upload land in at most 256 prefix dirs, so
+            // this replaces one dir fsync *per chunk* with ≤256 total.
+            let mut dirs: Vec<PathBuf> = paths
+                .iter()
+                .filter_map(|p| p.parent().map(Path::to_path_buf))
+                .collect();
+            dirs.sort_unstable();
+            dirs.dedup();
+
+            // Files first (hard requirement), then dirents (best-effort,
+            // same tier as fsync_parent_dir).
+            fsync_paths_parallel(paths, true).await?;
+            fsync_paths_parallel(dirs, false).await?;
+            Ok(())
+        })
+    }
+
     fn get_blob_stream(
         &self,
         hash: &str,
@@ -382,3 +493,93 @@ impl BlobStorageBackend for LocalBlobBackend {
         Some(self.blob_path(hash))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures::StreamExt;
+    use tempfile::TempDir;
+
+    /// 64-char fake hash with the given 2-char prefix (selects the prefix dir).
+    fn fake_hash(prefix: &str) -> String {
+        format!("{prefix}{}", "0".repeat(62))
+    }
+
+    async fn read_blob(backend: &LocalBlobBackend, hash: &str) -> Vec<u8> {
+        let mut stream = backend.get_blob_stream(hash).await.unwrap();
+        let mut data = Vec::new();
+        while let Some(chunk) = stream.next().await {
+            data.extend_from_slice(&chunk.unwrap());
+        }
+        data
+    }
+
+    #[tokio::test]
+    async fn unsynced_write_then_sync_blobs_roundtrip() {
+        let tmp = TempDir::new().unwrap();
+        let backend = LocalBlobBackend::new(tmp.path());
+        backend.initialize().await.unwrap();
+
+        // Two different prefixes → exercises the distinct-parent-dir dedup.
+        let h1 = fake_hash("aa");
+        let h2 = fake_hash("bb");
+        backend
+            .put_blob_from_bytes_unsynced(&h1, Bytes::from_static(b"chunk one"))
+            .await
+            .unwrap();
+        backend
+            .put_blob_from_bytes_unsynced(&h2, Bytes::from_static(b"chunk two"))
+            .await
+            .unwrap();
+
+        backend.sync_blobs(&[h1.clone(), h2.clone()]).await.unwrap();
+
+        assert!(backend.blob_exists(&h1).await.unwrap());
+        assert!(backend.blob_exists(&h2).await.unwrap());
+        assert_eq!(read_blob(&backend, &h1).await, b"chunk one");
+        assert_eq!(read_blob(&backend, &h2).await, b"chunk two");
+    }
+
+    #[tokio::test]
+    async fn unsynced_write_is_idempotent() {
+        let tmp = TempDir::new().unwrap();
+        let backend = LocalBlobBackend::new(tmp.path());
+        backend.initialize().await.unwrap();
+
+        let hash = fake_hash("cc");
+        let size1 = backend
+            .put_blob_from_bytes_unsynced(&hash, Bytes::from_static(b"same content"))
+            .await
+            .unwrap();
+        let size2 = backend
+            .put_blob_from_bytes_unsynced(&hash, Bytes::from_static(b"same content"))
+            .await
+            .unwrap();
+
+        assert_eq!(size1, size2);
+        assert_eq!(read_blob(&backend, &hash).await, b"same content");
+    }
+
+    #[tokio::test]
+    async fn sync_blobs_fails_on_missing_blob() {
+        let tmp = TempDir::new().unwrap();
+        let backend = LocalBlobBackend::new(tmp.path());
+        backend.initialize().await.unwrap();
+
+        let missing = fake_hash("dd");
+        assert!(
+            backend.sync_blobs(&[missing]).await.is_err(),
+            "sweeping a never-written blob must fail — the caller would \
+             otherwise insert a PG row for a chunk that doesn't exist"
+        );
+    }
+
+    #[tokio::test]
+    async fn sync_blobs_empty_is_noop() {
+        let tmp = TempDir::new().unwrap();
+        let backend = LocalBlobBackend::new(tmp.path());
+        backend.initialize().await.unwrap();
+
+        backend.sync_blobs(&[]).await.unwrap();
+    }
+}