Bradley Nelson
0a93123859
These example targets landed unformatted on main and fail the Rustfmt CI check (`cargo fmt --all --check`); reformat them so this PR's checks pass. No logic changes.
302 lines
11 KiB
Rust
302 lines
11 KiB
Rust
//! Blob download read-ahead benchmark — `read_prefetch()` / `buffered(N)`.
|
|
//!
|
|
//! Isolates the ONE variable the local-backend change touches: the chunk
|
|
//! read-ahead depth fed to `buffered(N)` when reassembling a CDC file on the
|
|
//! download path (`DedupService::stream_chunks`). It rebuilds the *exact*
|
|
//! production combinator —
|
|
//!
|
|
//! `stream::iter(hashes).map(get_blob_stream).buffered(N).try_flatten()`
|
|
//!
|
|
//! — over a REAL `LocalBlobBackend` whose chunk files are scattered across the
|
|
//! 256 hash-prefix directories exactly like production, then drains it and
|
|
//! reports throughput. The `N = 1` row is the current production behaviour
|
|
//! ("antes"); the higher-N rows are the candidate change ("después").
|
|
//!
|
|
//! The outcome is workload-dependent (the trait doc for `read_prefetch` argues
|
|
//! local should stay at 1), so the bench sweeps the two axes that decide it:
|
|
//! • Consumer speed — `unthrottled` (disk-bound: a localhost / LAN client that
|
|
//! drains as fast as the disk delivers) vs `throttled@<MB/s>` (network-bound:
|
|
//! a real remote client where the socket, not the disk, is the bottleneck —
|
|
//! this is where overlapping the next chunk's open+read with the current
|
|
//! chunk's socket drain is supposed to pay off).
|
|
//! • Page-cache state — `warm` (re-read, no disk I/O) vs `cold`
|
|
//! (`posix_fadvise(DONTNEED)` evicts each chunk file first, Linux only —
|
|
//! where concurrent opens on scattered files can instead cause seek
|
|
//! contention). `cold` is best-effort: on tmpfs/overlayfs the eviction is a
|
|
//! no-op and `cold` ≈ `warm` (noted in the output).
|
|
//!
|
|
//! Run (no Postgres needed):
|
|
//! cargo run --release --features bench --example bench_blob_prefetch
|
|
//! Tunables (env):
|
|
//! BENCH_FILE_MB (256) total blob size
|
|
//! BENCH_CHUNK_KB (256) per-chunk size (matches CDC_AVG_CHUNK)
|
|
//! BENCH_PREFETCH ("1,2,4,8,16")
|
|
//! BENCH_THROTTLE_MBPS ("0,300,100") 0 = unthrottled; each value = a throttled run
|
|
//! BENCH_REPS (5) repetitions per cell; median reported
|
|
//! BENCH_COLD (1) also run cold-cache rows (Linux x86-64 only)
|
|
|
|
use std::env;
|
|
use std::path::PathBuf;
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use bytes::Bytes;
|
|
use futures::{StreamExt, TryStreamExt, stream};
|
|
|
|
use oxicloud::application::ports::blob_storage_ports::BlobStorageBackend;
|
|
use oxicloud::infrastructure::services::local_blob_backend::LocalBlobBackend;
|
|
|
|
fn env_or<T: std::str::FromStr>(key: &str, default: T) -> T {
|
|
env::var(key)
|
|
.ok()
|
|
.and_then(|v| v.parse().ok())
|
|
.unwrap_or(default)
|
|
}
|
|
|
|
fn env_list_usize(key: &str, default: &[usize]) -> Vec<usize> {
|
|
env::var(key)
|
|
.ok()
|
|
.map(|s| {
|
|
s.split(',')
|
|
.filter_map(|x| x.trim().parse().ok())
|
|
.collect::<Vec<_>>()
|
|
})
|
|
.filter(|v: &Vec<usize>| !v.is_empty())
|
|
.unwrap_or_else(|| default.to_vec())
|
|
}
|
|
|
|
// ── cold-cache eviction (Linux x86-64 only, best-effort) ─────────────────────
|
|
#[cfg(all(target_os = "linux", target_pointer_width = "64"))]
|
|
unsafe extern "C" {
|
|
fn posix_fadvise(fd: i32, offset: i64, len: i64, advice: i32) -> i32;
|
|
}
|
|
#[cfg(all(target_os = "linux", target_pointer_width = "64"))]
|
|
const POSIX_FADV_DONTNEED: i32 = 4;
|
|
|
|
#[cfg(all(target_os = "linux", target_pointer_width = "64"))]
|
|
fn evict(paths: &[PathBuf]) {
|
|
use std::os::unix::io::AsRawFd;
|
|
for p in paths {
|
|
if let Ok(f) = std::fs::File::open(p) {
|
|
// len = 0 → "from offset to end of file" (the whole blob).
|
|
unsafe {
|
|
posix_fadvise(f.as_raw_fd(), 0, 0, POSIX_FADV_DONTNEED);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#[cfg(not(all(target_os = "linux", target_pointer_width = "64")))]
|
|
fn evict(_paths: &[PathBuf]) {}
|
|
|
|
#[cfg(all(target_os = "linux", target_pointer_width = "64"))]
|
|
const COLD_SUPPORTED: bool = true;
|
|
#[cfg(not(all(target_os = "linux", target_pointer_width = "64")))]
|
|
const COLD_SUPPORTED: bool = false;
|
|
|
|
/// Fill `buf` with distinct, well-distributed bytes (xorshift64 seeded per
|
|
/// chunk) so every chunk hashes to a different BLAKE3 → scattered across the
|
|
/// 256 prefix dirs, matching production's content-addressed layout.
|
|
fn fill_chunk(buf: &mut [u8], seed: u64) {
|
|
let mut s = seed ^ 0x9E37_79B9_7F4A_7C15;
|
|
let mut i = 0;
|
|
while i + 8 <= buf.len() {
|
|
s ^= s << 13;
|
|
s ^= s >> 7;
|
|
s ^= s << 17;
|
|
buf[i..i + 8].copy_from_slice(&s.to_le_bytes());
|
|
i += 8;
|
|
}
|
|
while i < buf.len() {
|
|
s ^= s << 13;
|
|
s ^= s >> 7;
|
|
s ^= s << 17;
|
|
buf[i] = s as u8;
|
|
i += 1;
|
|
}
|
|
}
|
|
|
|
/// Drain the production reassembly pipeline once; return bytes read.
|
|
/// `throttle_bps == 0` means unthrottled (drain as fast as possible).
|
|
async fn run_once(
|
|
backend: Arc<dyn BlobStorageBackend>,
|
|
hashes: Vec<String>,
|
|
prefetch: usize,
|
|
throttle_bps: f64,
|
|
) -> u64 {
|
|
let backend_for_map = backend.clone();
|
|
let mut byte_stream = stream::iter(hashes)
|
|
.map(move |hash| {
|
|
let b = backend_for_map.clone();
|
|
async move { b.get_blob_stream(&hash).await }
|
|
})
|
|
.buffered(prefetch.max(1))
|
|
.map(|r| r.map_err(std::io::Error::other))
|
|
.try_flatten();
|
|
|
|
let mut total: u64 = 0;
|
|
// Coarse token-bucket: only sleep once the accumulated owed time clears a
|
|
// 2 ms floor, so the throttle models a rate-limited socket without drowning
|
|
// the measurement in sub-ms timer noise.
|
|
let per_byte_secs = if throttle_bps > 0.0 {
|
|
1.0 / throttle_bps
|
|
} else {
|
|
0.0
|
|
};
|
|
let mut owed = Duration::ZERO;
|
|
|
|
while let Some(item) = byte_stream.next().await {
|
|
let chunk = item.expect("blob stream item");
|
|
total += chunk.len() as u64;
|
|
if per_byte_secs > 0.0 {
|
|
owed += Duration::from_secs_f64(chunk.len() as f64 * per_byte_secs);
|
|
if owed >= Duration::from_millis(2) {
|
|
tokio::time::sleep(owed).await;
|
|
owed = Duration::ZERO;
|
|
}
|
|
}
|
|
}
|
|
total
|
|
}
|
|
|
|
#[tokio::main(flavor = "multi_thread")]
|
|
async fn main() {
|
|
let file_mb: usize = env_or("BENCH_FILE_MB", 256);
|
|
let chunk_kb: usize = env_or("BENCH_CHUNK_KB", 256);
|
|
let prefetches = env_list_usize("BENCH_PREFETCH", &[1, 2, 4, 8, 16]);
|
|
let throttles_mbps = env_list_usize("BENCH_THROTTLE_MBPS", &[0, 300, 100]);
|
|
let reps: usize = env_or("BENCH_REPS", 5);
|
|
let want_cold: bool = env_or::<u8>("BENCH_COLD", 1) != 0;
|
|
|
|
let chunk_bytes = chunk_kb * 1024;
|
|
let total_bytes = file_mb * 1024 * 1024;
|
|
let n_chunks = total_bytes.div_ceil(chunk_bytes);
|
|
|
|
let tmp = tempfile::tempdir().expect("tempdir");
|
|
let backend_local = LocalBlobBackend::new(tmp.path());
|
|
backend_local.initialize().await.expect("init backend");
|
|
|
|
// ── Build the blob: write n_chunks distinct content-addressed chunk files.
|
|
let mut hashes: Vec<String> = Vec::with_capacity(n_chunks);
|
|
let mut paths: Vec<PathBuf> = Vec::with_capacity(n_chunks);
|
|
let mut buf = vec![0u8; chunk_bytes];
|
|
let build_start = Instant::now();
|
|
for i in 0..n_chunks {
|
|
fill_chunk(&mut buf, i as u64);
|
|
let data = Bytes::copy_from_slice(&buf);
|
|
let hash = blake3::hash(&data).to_hex().to_string();
|
|
backend_local
|
|
.put_blob_from_bytes(&hash, data)
|
|
.await
|
|
.expect("put blob");
|
|
paths.push(backend_local.blob_path(&hash));
|
|
hashes.push(hash);
|
|
}
|
|
let backend: Arc<dyn BlobStorageBackend> = Arc::new(backend_local);
|
|
let actual_bytes: u64 = (n_chunks * chunk_bytes) as u64;
|
|
|
|
println!("\n############################################################");
|
|
println!("# Blob download read-ahead (read_prefetch / buffered(N))");
|
|
println!(
|
|
"# blob: {} MiB in {} chunks of {} KiB (built in {:.1}s)",
|
|
file_mb,
|
|
n_chunks,
|
|
chunk_kb,
|
|
build_start.elapsed().as_secs_f64()
|
|
);
|
|
println!(
|
|
"# production LocalBlobBackend.read_prefetch() = {}",
|
|
backend.read_prefetch()
|
|
);
|
|
println!(
|
|
"# reps/cell: {reps} (median MB/s reported) cold-cache: {}",
|
|
{
|
|
if !COLD_SUPPORTED {
|
|
"unsupported (non-Linux) → warm only"
|
|
} else if want_cold {
|
|
"yes (posix_fadvise DONTNEED, best-effort)"
|
|
} else {
|
|
"disabled (BENCH_COLD=0)"
|
|
}
|
|
}
|
|
);
|
|
println!("# N=1 is current production ('antes'); higher N is the candidate ('después')");
|
|
println!("############################################################\n");
|
|
println!(
|
|
"| {:<22} | {:>8} | {:>9} | {:>8} | {:>9} |",
|
|
"scenario", "prefetch", "med MB/s", "min ms", "vs N=1"
|
|
);
|
|
println!(
|
|
"|{:-<24}|{:-<10}|{:-<11}|{:-<10}|{:-<11}|",
|
|
"", "", "", "", ""
|
|
);
|
|
|
|
let mb = actual_bytes as f64 / (1024.0 * 1024.0);
|
|
|
|
// cache states to test
|
|
let mut cache_states: Vec<&str> = vec!["warm"];
|
|
if want_cold && COLD_SUPPORTED {
|
|
cache_states.push("cold");
|
|
}
|
|
|
|
for &thr_mbps in &throttles_mbps {
|
|
let throttle_bps = thr_mbps as f64 * 1024.0 * 1024.0;
|
|
let thr_label = if thr_mbps == 0 {
|
|
"unthrottled".to_string()
|
|
} else {
|
|
format!("throttled@{}MB/s", thr_mbps)
|
|
};
|
|
|
|
for cache in &cache_states {
|
|
let scenario = format!("{}/{}", cache, thr_label);
|
|
let mut baseline_mbps: Option<f64> = None;
|
|
|
|
for &pf in &prefetches {
|
|
let mut samples_mbps: Vec<f64> = Vec::with_capacity(reps);
|
|
let mut min_ms = f64::MAX;
|
|
|
|
// one warmup (also primes warm-cache state)
|
|
let _ = run_once(backend.clone(), hashes.clone(), pf, throttle_bps).await;
|
|
|
|
for _ in 0..reps {
|
|
if *cache == "cold" {
|
|
evict(&paths);
|
|
}
|
|
let t = Instant::now();
|
|
let got = run_once(backend.clone(), hashes.clone(), pf, throttle_bps).await;
|
|
let secs = t.elapsed().as_secs_f64();
|
|
assert_eq!(got, actual_bytes, "short read");
|
|
samples_mbps.push(mb / secs);
|
|
min_ms = min_ms.min(secs * 1000.0);
|
|
}
|
|
|
|
samples_mbps.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
|
let med = samples_mbps[samples_mbps.len() / 2];
|
|
let delta = match baseline_mbps {
|
|
None => {
|
|
baseline_mbps = Some(med);
|
|
"—".to_string()
|
|
}
|
|
Some(base) => format!("{:+.1}%", (med / base - 1.0) * 100.0),
|
|
};
|
|
|
|
println!(
|
|
"| {:<22} | {:>8} | {:>9.1} | {:>8.1} | {:>9} |",
|
|
scenario, pf, med, min_ms, delta
|
|
);
|
|
}
|
|
println!(
|
|
"|{:-<24}|{:-<10}|{:-<11}|{:-<10}|{:-<11}|",
|
|
"", "", "", "", ""
|
|
);
|
|
}
|
|
}
|
|
|
|
println!(
|
|
"\nInterpretation: a '+x%' under 'vs N=1' is the read-ahead gain over current\n\
|
|
production for that scenario; a negative value is a regression. Network-bound\n\
|
|
rows (throttled) are the realistic remote-download case; unthrottled rows are\n\
|
|
disk-bound (localhost/LAN). Pick the smallest N that wins the throttled rows\n\
|
|
without regressing the disk-bound/cold rows.\n"
|
|
);
|
|
}
|