perf: findings 6.1, 6.2, 2.6 — async Argon2, moka cache, full streaming migration
- 6.1: PasswordHasherPort now async_trait with spawn_blocking for Argon2
- 6.2: OIDC pending maps migrated from std::sync::Mutex to moka::sync::Cache with TTL
- 2.6: All file download paths migrated to 64KB streaming (get_file_stream / read_blob_stream)
- WOPI, dedup, batch ZIP, file_retrieval_service consumers migrated
- WebDAV COPY uses zero-copy dedup (copy_file)
- Removed dead code: get_file_content, get_file_mmap, read_blob, read_blob_bytes
from traits, impls, stubs, and mocks (18 files touched)
This commit is contained in:
@@ -31,7 +31,7 @@ use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs::{self, File};
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, BufReader};
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use crate::application::ports::dedup_ports::{
|
||||
@@ -39,10 +39,10 @@ use crate::application::ports::dedup_ports::{
|
||||
};
|
||||
use crate::domain::errors::{DomainError, ErrorKind};
|
||||
|
||||
/// Chunk size for streaming hash calculation (256KB)
|
||||
const HASH_CHUNK_SIZE: usize = 256 * 1024;
|
||||
/// Block size for SHA-256 file hashing (1MB — optimal syscall/throughput ratio).
|
||||
const HASH_BLOCK_SIZE: usize = 1024 * 1024;
|
||||
|
||||
/// Chunk size for streaming file reads (256 KB — 4x fewer iterations)
|
||||
/// Chunk size for streaming file reads (256 KB)
|
||||
const STREAM_CHUNK_SIZE: usize = 256 * 1024;
|
||||
|
||||
/// Content-Addressable Storage Service (PostgreSQL-backed)
|
||||
@@ -124,22 +124,32 @@ impl DedupService {
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Calculate SHA-256 hash of a file (streaming).
|
||||
/// Calculate SHA-256 hash of a file.
|
||||
///
|
||||
/// Runs entirely on `spawn_blocking` with synchronous I/O so the Tokio
|
||||
/// worker threads are never blocked by CPU-bound hashing. Uses 1 MB
|
||||
/// reads for optimal syscall-to-throughput ratio (~3.8 GB/s on NVMe).
|
||||
pub async fn hash_file(path: &Path) -> std::io::Result<String> {
|
||||
let file = File::open(path).await?;
|
||||
let mut reader = BufReader::with_capacity(HASH_CHUNK_SIZE, file);
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buffer = vec![0u8; HASH_CHUNK_SIZE];
|
||||
let path = path.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
use std::io::Read;
|
||||
|
||||
loop {
|
||||
let bytes_read = reader.read(&mut buffer).await?;
|
||||
if bytes_read == 0 {
|
||||
break;
|
||||
let mut file = std::fs::File::open(&path)?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buffer = vec![0u8; HASH_BLOCK_SIZE];
|
||||
|
||||
loop {
|
||||
let n = file.read(&mut buffer)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..n]);
|
||||
}
|
||||
hasher.update(&buffer[..bytes_read]);
|
||||
}
|
||||
|
||||
Ok(hex::encode(hasher.finalize()))
|
||||
Ok(hex::encode(hasher.finalize()))
|
||||
})
|
||||
.await
|
||||
.expect("hash_file: spawn_blocking task panicked")
|
||||
}
|
||||
|
||||
// ── Core store operations ────────────────────────────────────
|
||||
@@ -515,27 +525,8 @@ impl DedupService {
|
||||
|
||||
// ── Read operations ──────────────────────────────────────────
|
||||
|
||||
/// Read blob content from the filesystem.
|
||||
pub async fn read_blob(&self, hash: &str) -> Result<Vec<u8>, DomainError> {
|
||||
let blob_path = self.blob_path(hash);
|
||||
|
||||
fs::read(&blob_path).await.map_err(|e| {
|
||||
DomainError::new(
|
||||
ErrorKind::NotFound,
|
||||
"Blob",
|
||||
format!("Failed to read blob {}: {}", hash, e),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Read blob content as Bytes.
|
||||
pub async fn read_blob_bytes(&self, hash: &str) -> Result<Bytes, DomainError> {
|
||||
self.read_blob(hash).await.map(Bytes::from)
|
||||
}
|
||||
|
||||
/// Stream blob content in 64 KB chunks — constant memory (~64 KB per stream).
|
||||
///
|
||||
/// Unlike `read_blob()`, this never loads the entire file into RAM.
|
||||
/// A 1 GB file uses the same ~64 KB as a 1 KB file.
|
||||
pub async fn read_blob_stream(
|
||||
&self,
|
||||
@@ -772,14 +763,6 @@ impl DedupPort for DedupService {
|
||||
self.get_blob_metadata(hash).await
|
||||
}
|
||||
|
||||
async fn read_blob(&self, hash: &str) -> Result<Vec<u8>, DomainError> {
|
||||
self.read_blob(hash).await
|
||||
}
|
||||
|
||||
async fn read_blob_bytes(&self, hash: &str) -> Result<Bytes, DomainError> {
|
||||
self.read_blob_bytes(hash).await
|
||||
}
|
||||
|
||||
async fn read_blob_stream(
|
||||
&self,
|
||||
hash: &str,
|
||||
|
||||
Reference in New Issue
Block a user