diff --git a/Cargo.lock b/Cargo.lock index c2f03b67..8a52999b 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,6 +278,7 @@ dependencies = [ "cfg-if", "constant_time_eq", "cpufeatures", + "memmap2", "rayon-core", ] @@ -1608,6 +1609,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "mimalloc" version = "0.1.48" diff --git a/Cargo.toml b/Cargo.toml index 80c6cc5a..18f5e1f7 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ image = { version = "0.25", default-features = false, features = ["jpeg", "png", kamadak-exif = "0.5" md-5 = "0.10" sha2 = "0.10.9" -blake3 = { version = "1.8.3", features = ["rayon"] } +blake3 = { version = "1.8.3", features = ["rayon", "mmap"] } hex = "0.4.3" http-body-util = "0.1.3" percent-encoding = "2.3" diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 907cf453..345db8ef 100755 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -48,13 +48,6 @@ use crate::application::ports::dedup_ports::{ }; use crate::domain::errors::{DomainError, ErrorKind}; -/// Block size for BLAKE3 file hashing (1MB — optimal syscall/throughput ratio). -const HASH_BLOCK_SIZE: usize = 1024 * 1024; - -/// Files larger than this threshold use multithreaded BLAKE3 hashing via -/// `update_rayon()`, which splits the work across all available cores. -const RAYON_HASH_THRESHOLD: u64 = 10 * 1024 * 1024; // 10 MB - /// Chunk size for streaming file reads (256 KB) const STREAM_CHUNK_SIZE: usize = 256 * 1024; @@ -175,12 +168,12 @@ impl DedupService { // ── Hash helpers ───────────────────────────────────────────── - /// Calculate BLAKE3 hash of content (~5× faster than SHA-256). + /// Calculate BLAKE3 hash of in-memory content (~5× faster than SHA-256). /// - /// For buffers larger than 10 MB the computation is parallelised across + /// For buffers larger than 128 KB the computation is parallelised across /// all available cores via `update_rayon()`. pub fn hash_bytes(content: &[u8]) -> String { - if content.len() as u64 > RAYON_HASH_THRESHOLD { + if content.len() > 128 * 1024 { let mut hasher = blake3::Hasher::new(); hasher.update_rayon(content); hasher.finalize().to_hex().to_string() @@ -194,33 +187,16 @@ impl DedupService { /// Runs entirely on `spawn_blocking` with synchronous I/O so the Tokio /// worker threads are never blocked by CPU-bound hashing. /// - /// For files larger than 10 MB the hash is computed with `update_rayon()`, - /// which splits the work across all available cores. Smaller files use - /// sequential 1 MB reads for optimal syscall-to-throughput ratio. + /// Uses memory-mapped I/O (`update_mmap_rayon`) which avoids loading the + /// entire file into the heap. The OS pages in data on demand and BLAKE3 + /// parallelises the computation across all available cores via rayon. + /// Peak RAM for a 500 MB file is only a few MB of active pages instead + /// of the full 500 MB. pub async fn hash_file(path: &Path) -> std::io::Result { let path = path.to_path_buf(); tokio::task::spawn_blocking(move || { - let file_size = std::fs::metadata(&path)?.len(); let mut hasher = blake3::Hasher::new(); - - if file_size > RAYON_HASH_THRESHOLD { - // Large file: read into memory and hash with all cores - let content = std::fs::read(&path)?; - hasher.update_rayon(&content); - } else { - // Small file: sequential streaming with 1 MB reads - use std::io::Read; - let mut file = std::fs::File::open(&path)?; - let mut buffer = vec![0u8; HASH_BLOCK_SIZE]; - loop { - let n = file.read(&mut buffer)?; - if n == 0 { - break; - } - hasher.update(&buffer[..n]); - } - } - + hasher.update_mmap_rayon(&path)?; Ok(hasher.finalize().to_hex().to_string()) }) .await