perf: round 17 — dedup ingest/verify hash-clone purge, CardDAV vCard TYPE tokens
Targets the content-addressable dedup write path (the ROUND15-deferred "dedup_service hash-String re-allocations") from both ends — the streaming ingest loop and the delta-commit verification read — plus a CardDAV vCard micro-cut. Every change is benchmark-gated with a hard rollback rule; no PostgreSQL needed for any arm (benches/ROUND17.md). Backend (counting-allocator, examples/bench_round17_micro.rs): - D2 chunk-ingest (store_from_stream, the hottest write path — every chunk of every upload): the 64-char hex hash String was allocated 3x per chunk (to_hex + chunk_hashes clone + session_seen insert-clone, the last dropped on a duplicate). The intra-upload dedup set now keys on the raw 32-byte BLAKE3 digest ([u8;32], Copy, no heap) and the manifest push is branch-split so a duplicate moves the hex in: 3 -> 2 allocs/new chunk, 3 -> 1/duplicate. Measured 214 -> 149 allocs/op (1.14x wall) on a 64-chunk 1-in-2-dup batch; smaller/faster set too (32B inline keys vs 64B heap Strings). - D1 hash_chunk_sequence (delta-commit verification): took chunks by &[(String,u64)] and fed the backend stream with iter().cloned(), re-cloning every chunk hash a second time on top of the owned Vec the caller already built. Take the Vec by value + into_iter(): 65 -> 0 internal allocs/op, ~2.5us of clone work removed per verify. - V1 vCard TYPE tokens (contact_to_vcard + generate_vcard, 5 sites): each EMAIL/TEL/ADR TYPE= param used ty.to_uppercase() — a throw-away String per token per contact. New shared fmt::push_upper writes the upper-cased chars straight into the buffer (byte-identical to str::to_uppercase, unit-tested): 13 -> 5 allocs/op, 1.19x wall. Gates: each section asserts byte/-value equivalence (D2 the ordered manifest + sizes + write-set; D1 the removed clone is a pure copy; V1 the full vCard) and exits non-zero if an AFTER arm fails to reduce allocations. push_upper is unit-tested byte-equal to str::to_uppercase (fmt::tests). Verified end-to-end: cargo fmt clean, clippy --release --all-targets --features bench -D warnings clean, and the harness prints GATE PASS against the built release lib. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XMmt7vNETYUbEG3Hc17LDx
This commit is contained in:
@@ -970,28 +970,27 @@ pub fn contact_to_vcard(contact: &ContactDto) -> String {
|
||||
}
|
||||
|
||||
for email in &contact.email {
|
||||
let _ = write!(
|
||||
vcard,
|
||||
"EMAIL;TYPE={}:{}\r\n",
|
||||
email.r#type.to_uppercase(),
|
||||
email.email
|
||||
);
|
||||
vcard.push_str("EMAIL;TYPE=");
|
||||
crate::common::fmt::push_upper(&mut vcard, &email.r#type);
|
||||
vcard.push(':');
|
||||
vcard.push_str(&email.email);
|
||||
vcard.push_str("\r\n");
|
||||
}
|
||||
|
||||
for phone in &contact.phone {
|
||||
let _ = write!(
|
||||
vcard,
|
||||
"TEL;TYPE={}:{}\r\n",
|
||||
phone.r#type.to_uppercase(),
|
||||
phone.number
|
||||
);
|
||||
vcard.push_str("TEL;TYPE=");
|
||||
crate::common::fmt::push_upper(&mut vcard, &phone.r#type);
|
||||
vcard.push(':');
|
||||
vcard.push_str(&phone.number);
|
||||
vcard.push_str("\r\n");
|
||||
}
|
||||
|
||||
for addr in &contact.address {
|
||||
vcard.push_str("ADR;TYPE=");
|
||||
crate::common::fmt::push_upper(&mut vcard, &addr.r#type);
|
||||
let _ = write!(
|
||||
vcard,
|
||||
"ADR;TYPE={}:;;{};{};{};{};{}\r\n",
|
||||
addr.r#type.to_uppercase(),
|
||||
":;;{};{};{};{};{}\r\n",
|
||||
addr.street.as_deref().unwrap_or(""),
|
||||
addr.city.as_deref().unwrap_or(""),
|
||||
addr.state.as_deref().unwrap_or(""),
|
||||
|
||||
@@ -283,12 +283,11 @@ impl ContactService {
|
||||
|
||||
// Email addresses
|
||||
for email in contact.email() {
|
||||
let _ = write!(
|
||||
vcard,
|
||||
"EMAIL;TYPE={}:{}\r\n",
|
||||
email.r#type.to_uppercase(),
|
||||
email.email
|
||||
);
|
||||
vcard.push_str("EMAIL;TYPE=");
|
||||
crate::common::fmt::push_upper(&mut vcard, &email.r#type);
|
||||
vcard.push(':');
|
||||
vcard.push_str(&email.email);
|
||||
vcard.push_str("\r\n");
|
||||
}
|
||||
|
||||
// Phone numbers
|
||||
@@ -305,17 +304,18 @@ impl ContactService {
|
||||
|
||||
// Addresses
|
||||
for addr in contact.address() {
|
||||
let addr_type = addr.r#type.to_uppercase();
|
||||
let street = addr.street.as_deref().unwrap_or_default();
|
||||
let city = addr.city.as_deref().unwrap_or_default();
|
||||
let state = addr.state.as_deref().unwrap_or_default();
|
||||
let postal_code = addr.postal_code.as_deref().unwrap_or_default();
|
||||
let country = addr.country.as_deref().unwrap_or_default();
|
||||
|
||||
vcard.push_str("ADR;TYPE=");
|
||||
crate::common::fmt::push_upper(&mut vcard, &addr.r#type);
|
||||
let _ = write!(
|
||||
vcard,
|
||||
"ADR;TYPE={}:;;{};{};{};{};{}\r\n",
|
||||
addr_type, street, city, state, postal_code, country
|
||||
":;;{};{};{};{};{}\r\n",
|
||||
street, city, state, postal_code, country
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -398,7 +398,7 @@ impl DeltaUploadService {
|
||||
let verification = self
|
||||
.dedup
|
||||
.hash_chunk_sequence(
|
||||
&request
|
||||
request
|
||||
.chunks
|
||||
.iter()
|
||||
.map(|c| (c.h.clone(), c.s))
|
||||
|
||||
@@ -210,6 +210,22 @@ pub fn hex_lower(bytes: &[u8]) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Append the upper-cased form of `s` to `buf` without a temporary `String`.
|
||||
///
|
||||
/// Byte-identical to `buf.push_str(&s.to_uppercase())` — same
|
||||
/// `char::to_uppercase` expansion (incl. ß → SS, ff → FF) — but writes straight
|
||||
/// into the caller's buffer. The vCard emit path (`contact_to_vcard`,
|
||||
/// `generate_vcard`) formats an `EMAIL`/`TEL`/`ADR` `TYPE=` token per line, and
|
||||
/// the old `write!(…, "{}", ty.to_uppercase())` heap-allocated one throw-away
|
||||
/// `String` per token per contact (benches/ROUND17.md §V1).
|
||||
pub fn push_upper(buf: &mut String, s: &str) {
|
||||
for c in s.chars() {
|
||||
for u in c.to_uppercase() {
|
||||
buf.push(u);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -232,6 +248,20 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// `push_upper` must match `push_str(&s.to_uppercase())` byte for byte,
|
||||
/// including multi-char upper-casings (ß → SS) and dotless-i.
|
||||
#[test]
|
||||
fn push_upper_matches_to_uppercase() {
|
||||
let cases = [
|
||||
"", "home", "WORK", "Cell", "voice", "x-custom", "café", "straße", "ff", "ı",
|
||||
];
|
||||
for s in cases {
|
||||
let mut got = String::new();
|
||||
push_upper(&mut got, s);
|
||||
assert_eq!(got, s.to_uppercase(), "push_upper differs for {s:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Edge-heavy corpus: epoch, single-digit day (padding!), leap day,
|
||||
/// end-of-year, DST-irrelevant midsummer, far future, max in-range.
|
||||
const CASES: [i64; 12] = [
|
||||
|
||||
@@ -912,7 +912,7 @@ impl DedupService {
|
||||
/// the declared one — the manifest's Range arithmetic depends on it.
|
||||
pub async fn hash_chunk_sequence(
|
||||
&self,
|
||||
chunks: &[(String, u64)],
|
||||
chunks: Vec<(String, u64)>,
|
||||
sniff_len: usize,
|
||||
) -> Result<(String, Vec<u8>), DomainError> {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
@@ -925,7 +925,7 @@ impl DedupService {
|
||||
// strictly in manifest order: `buffered` yields in input order.
|
||||
let prefetch = self.backend.read_prefetch().max(1);
|
||||
let backend = self.backend.clone();
|
||||
let mut opened = futures::stream::iter(chunks.iter().cloned())
|
||||
let mut opened = futures::stream::iter(chunks)
|
||||
.map(move |(hash, declared_size)| {
|
||||
let backend = backend.clone();
|
||||
async move {
|
||||
@@ -1032,7 +1032,11 @@ impl DedupService {
|
||||
let mut total_size: u64 = 0;
|
||||
let mut chunk_hashes: Vec<String> = Vec::new();
|
||||
let mut chunk_sizes: Vec<u64> = Vec::new();
|
||||
let mut session_seen: HashSet<String> = HashSet::new();
|
||||
// Keyed on the raw 32-byte BLAKE3 digest (`Copy`, no heap) rather than
|
||||
// the 64-char hex String: the intra-upload dedup set no longer clones a
|
||||
// String per chunk, holds 32-byte inline keys, and hashes 32 bytes not
|
||||
// 64 on every membership test (benches/ROUND17.md §D2).
|
||||
let mut session_seen: HashSet<[u8; 32]> = HashSet::new();
|
||||
let mut pending: Vec<(String, Bytes)> = Vec::new();
|
||||
let mut pending_bytes: usize = 0;
|
||||
// Depth-1 settle pipeline: batch N settles on a spawned task while
|
||||
@@ -1077,12 +1081,19 @@ impl DedupService {
|
||||
// Per-chunk hashing is ≤ 1 MiB of BLAKE3 (< 1 ms) — cheaper than
|
||||
// a spawn_blocking round-trip per chunk.
|
||||
file_hasher.update(&data);
|
||||
let hash = blake3::hash(&data).to_hex().to_string();
|
||||
let digest = blake3::hash(&data);
|
||||
let hash = digest.to_hex().to_string();
|
||||
chunk_sizes.push(data.len() as u64);
|
||||
chunk_hashes.push(hash.clone());
|
||||
|
||||
if session_seen.insert(hash.clone()) {
|
||||
// The hex `hash` is materialised once. A genuinely new chunk needs
|
||||
// it in three places — the ordered manifest, the dedup set key and
|
||||
// the backend write — but the set keys on the raw digest (no clone),
|
||||
// so only `chunk_hashes` is cloned before `pending` takes the
|
||||
// original. A duplicate within this upload needs it only for the
|
||||
// manifest: the `else` moves it in, no clone (benches/ROUND17.md §D2).
|
||||
if session_seen.insert(*digest.as_bytes()) {
|
||||
pending_bytes += data.len();
|
||||
chunk_hashes.push(hash.clone());
|
||||
pending.push((hash, Bytes::from(data)));
|
||||
if pending.len() >= Self::FLUSH_MAX_CHUNKS || pending_bytes >= Self::FLUSH_MAX_BYTES
|
||||
{
|
||||
@@ -1111,6 +1122,10 @@ impl DedupService {
|
||||
}
|
||||
pending_bytes = 0;
|
||||
}
|
||||
} else {
|
||||
// Duplicate within this upload: only the ordered manifest needs
|
||||
// the hash. Move it in — no set/pending copy, zero extra allocs.
|
||||
chunk_hashes.push(hash);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4000,7 +4015,7 @@ mod delta_upload_integration_tests {
|
||||
.collect();
|
||||
|
||||
let (computed, head) = svc
|
||||
.hash_chunk_sequence(&sequence, 16)
|
||||
.hash_chunk_sequence(sequence.clone(), 16)
|
||||
.await
|
||||
.expect("verification read");
|
||||
assert_eq!(computed, file_hash, "recomputed hash must match");
|
||||
@@ -4015,7 +4030,7 @@ mod delta_upload_integration_tests {
|
||||
let mut lying = sequence.clone();
|
||||
lying[0].1 += 1;
|
||||
assert!(
|
||||
svc.hash_chunk_sequence(&lying, 0).await.is_err(),
|
||||
svc.hash_chunk_sequence(lying, 0).await.is_err(),
|
||||
"size lie must fail verification"
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user