perf: replace SHA-256 with BLAKE3 in WebDAV/WOPI, fix JOIN index usage, add LIMIT to favorites, remove dead lru crate, trim tokio features

- WebDAV PUT and WOPI PutFile: replace sha2::Sha256 with blake3::Hasher (~5x faster hashing, compatible with dedup service)
- Fix TEXT↔UUID JOIN anti-pattern in favorites and recent_items repos (enables PK index usage)
- Add LIMIT 500 to get_favorites query to prevent unbounded memory allocation
- Remove unused lru crate from Cargo.toml (superseded by moka)
- Replace tokio features=["full"] with explicit feature list (removes signal, process, test-util)
This commit is contained in:
Dionisio
2026-03-02 02:13:08 +01:00
parent b06206207f
commit dd27872a8c
7 changed files with 16 additions and 51 deletions
Generated
+1 -34
View File
@@ -838,12 +838,6 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]] [[package]]
name = "form_urlencoded" name = "form_urlencoded"
version = "1.2.2" version = "1.2.2"
@@ -1074,7 +1068,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [ dependencies = [
"allocator-api2", "allocator-api2",
"equivalent", "equivalent",
"foldhash 0.1.5", "foldhash",
] ]
[[package]] [[package]]
@@ -1082,11 +1076,6 @@ name = "hashbrown"
version = "0.16.1" version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash 0.2.0",
]
[[package]] [[package]]
name = "hashlink" name = "hashlink"
@@ -1557,15 +1546,6 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lru"
version = "0.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593"
dependencies = [
"hashbrown 0.16.1",
]
[[package]] [[package]]
name = "lru-slab" name = "lru-slab"
version = "0.1.2" version = "0.1.2"
@@ -1833,7 +1813,6 @@ dependencies = [
"image", "image",
"infer", "infer",
"jsonwebtoken", "jsonwebtoken",
"lru",
"md5", "md5",
"mimalloc", "mimalloc",
"mime_guess", "mime_guess",
@@ -2590,16 +2569,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "signal-hook-registry"
version = "1.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
dependencies = [
"errno",
"libc",
]
[[package]] [[package]]
name = "signature" name = "signature"
version = "2.2.0" version = "2.2.0"
@@ -3043,9 +3012,7 @@ dependencies = [
"bytes", "bytes",
"libc", "libc",
"mio", "mio",
"parking_lot",
"pin-project-lite", "pin-project-lite",
"signal-hook-registry",
"socket2", "socket2",
"tokio-macros", "tokio-macros",
"windows-sys 0.61.2", "windows-sys 0.61.2",
+1 -2
View File
@@ -8,7 +8,7 @@ default-run = "oxicloud"
[dependencies] [dependencies]
mimalloc = { version = "0.1.48", default-features = false } mimalloc = { version = "0.1.48", default-features = false }
axum = { version = "0.8.8", features = ["multipart", "http1", "tokio", "macros"] } axum = { version = "0.8.8", features = ["multipart", "http1", "tokio", "macros"] }
tokio = { version = "1.49.0", features = ["full"] } tokio = { version = "1.49.0", features = ["rt-multi-thread", "macros", "io-util", "net", "time", "sync", "fs"] }
tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] } tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] }
tokio-stream = { version = "0.1.18", features = ["fs"] } tokio-stream = { version = "0.1.18", features = ["fs"] }
bytes = "1.11.1" bytes = "1.11.1"
@@ -37,7 +37,6 @@ rand_core = { version = "0.6", features = ["std", "getrandom"] }
hyper = { version = "1.8.1", features = ["full"] } hyper = { version = "1.8.1", features = ["full"] }
quick-xml = "0.39.0" quick-xml = "0.39.0"
dotenvy = "0.15.7" dotenvy = "0.15.7"
lru = "0.16.3"
moka = { version = "0.12", features = ["future", "sync"] } moka = { version = "0.12", features = ["future", "sync"] }
http-range-header = "0.4" http-range-header = "0.4"
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] } image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
@@ -39,11 +39,12 @@ impl FavoritesRepositoryPort for FavoritesPgRepository {
COALESCE(f.updated_at, fld.updated_at) AS "modified_at" COALESCE(f.updated_at, fld.updated_at) AS "modified_at"
FROM auth.user_favorites uf FROM auth.user_favorites uf
LEFT JOIN storage.files f ON uf.item_type = 'file' LEFT JOIN storage.files f ON uf.item_type = 'file'
AND uf.item_id = f.id::TEXT AND f.id = uf.item_id::UUID
LEFT JOIN storage.folders fld ON uf.item_type = 'folder' LEFT JOIN storage.folders fld ON uf.item_type = 'folder'
AND uf.item_id = fld.id::TEXT AND fld.id = uf.item_id::UUID
WHERE uf.user_id = $1::TEXT WHERE uf.user_id = $1::TEXT
ORDER BY uf.created_at DESC ORDER BY uf.created_at DESC
LIMIT 500
"#, "#,
) )
.bind(user_uuid) .bind(user_uuid)
@@ -38,9 +38,9 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
COALESCE(f.folder_id::TEXT, fld.parent_id::TEXT) AS "parent_id" COALESCE(f.folder_id::TEXT, fld.parent_id::TEXT) AS "parent_id"
FROM auth.user_recent_files ur FROM auth.user_recent_files ur
LEFT JOIN storage.files f ON ur.item_type = 'file' LEFT JOIN storage.files f ON ur.item_type = 'file'
AND ur.item_id = f.id::TEXT AND f.id = ur.item_id::UUID
LEFT JOIN storage.folders fld ON ur.item_type = 'folder' LEFT JOIN storage.folders fld ON ur.item_type = 'folder'
AND ur.item_id = fld.id::TEXT AND fld.id = ur.item_id::UUID
WHERE ur.user_id = $1::TEXT WHERE ur.user_id = $1::TEXT
ORDER BY ur.accessed_at DESC ORDER BY ur.accessed_at DESC
LIMIT $2 LIMIT $2
+2 -2
View File
@@ -268,7 +268,7 @@ impl DedupService {
/// never held during disk I/O. /// never held during disk I/O.
/// ///
/// If `pre_computed_hash` is `Some`, the file will NOT be re-read for /// If `pre_computed_hash` is `Some`, the file will NOT be re-read for
/// SHA-256 — saving one full sequential read (the biggest I/O win). /// BLAKE3 — saving one full sequential read (the biggest I/O win).
pub async fn store_from_file( pub async fn store_from_file(
&self, &self,
source_path: &Path, source_path: &Path,
@@ -615,7 +615,7 @@ impl DedupService {
/// of `VERIFY_CONCURRENCY` using `buffer_unordered`. /// of `VERIFY_CONCURRENCY` using `buffer_unordered`.
pub async fn verify_integrity(&self) -> Result<Vec<String>, DomainError> { pub async fn verify_integrity(&self) -> Result<Vec<String>, DomainError> {
/// Max blobs verified concurrently. Each spawns a blocking /// Max blobs verified concurrently. Each spawns a blocking
/// thread for SHA-256 so this also caps blocking-pool pressure. /// thread for BLAKE3 so this also caps blocking-pool pressure.
const VERIFY_CONCURRENCY: usize = 16; const VERIFY_CONCURRENCY: usize = 16;
let mut row_stream = sqlx::query_as::<_, (String, i64)>( let mut row_stream = sqlx::query_as::<_, (String, i64)>(
@@ -632,7 +632,7 @@ async fn handle_head(
* Handles PUT requests to create or update files. * Handles PUT requests to create or update files.
* *
* **Streaming implementation**: the request body is spooled to a temp file * **Streaming implementation**: the request body is spooled to a temp file
* with incremental SHA-256 hashing. Peak RAM usage is ~256 KB regardless * with incremental BLAKE3 hashing. Peak RAM usage is ~256 KB regardless
* of file size. The temp file is then atomically moved into blob storage * of file size. The temp file is then atomically moved into blob storage
* via `update_file_streaming`. * via `update_file_streaming`.
* *
@@ -647,7 +647,6 @@ async fn handle_put(
path: String, path: String,
) -> Result<Response<Body>, AppError> { ) -> Result<Response<Body>, AppError> {
use http_body_util::BodyStream; use http_body_util::BodyStream;
use sha2::{Digest, Sha256};
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
use tokio_stream::StreamExt; use tokio_stream::StreamExt;
@@ -679,7 +678,7 @@ async fn handle_put(
.await .await
.map_err(|e| AppError::internal_error(format!("Failed to open temp file: {}", e)))?; .map_err(|e| AppError::internal_error(format!("Failed to open temp file: {}", e)))?;
let mut hasher = Sha256::new(); let mut hasher = blake3::Hasher::new();
let mut total_bytes: usize = 0; let mut total_bytes: usize = 0;
let mut stream = BodyStream::new(req.into_body()); let mut stream = BodyStream::new(req.into_body());
@@ -708,7 +707,7 @@ async fn handle_put(
.map_err(|e| AppError::internal_error(format!("Failed to flush temp file: {}", e)))?; .map_err(|e| AppError::internal_error(format!("Failed to flush temp file: {}", e)))?;
drop(file); drop(file);
let hash = hex::encode(hasher.finalize()); let hash = hasher.finalize().to_hex().to_string();
// ── Atomic store: temp file → dedup blob + DB metadata update ── // ── Atomic store: temp file → dedup blob + DB metadata update ──
let result = file_upload_service let result = file_upload_service
+4 -5
View File
@@ -155,7 +155,7 @@ async fn get_file(
/// POST /wopi/files/{file_id}/contents — PutFile /// POST /wopi/files/{file_id}/contents — PutFile
/// ///
/// **Streaming implementation**: the request body is spooled to a temp file /// **Streaming implementation**: the request body is spooled to a temp file
/// with incremental SHA-256 hashing. Peak RAM usage is ~256 KB regardless /// with incremental BLAKE3 hashing. Peak RAM usage is ~256 KB regardless
/// of file size (previously buffered the entire body as `Bytes`). /// of file size (previously buffered the entire body as `Bytes`).
async fn put_file( async fn put_file(
Path(file_id): Path<String>, Path(file_id): Path<String>,
@@ -165,7 +165,6 @@ async fn put_file(
req: Request<Body>, req: Request<Body>,
) -> Response { ) -> Response {
use http_body_util::BodyStream; use http_body_util::BodyStream;
use sha2::{Digest, Sha256};
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
use tokio_stream::StreamExt; use tokio_stream::StreamExt;
@@ -218,7 +217,7 @@ async fn put_file(
Err(_) => return StatusCode::NOT_FOUND.into_response(), Err(_) => return StatusCode::NOT_FOUND.into_response(),
}; };
// ── Streaming spool: body → temp file + incremental SHA-256 ── // ── Streaming spool: body → temp file + incremental BLAKE3 ──
let temp_file = match tempfile::NamedTempFile::new() { let temp_file = match tempfile::NamedTempFile::new() {
Ok(f) => f, Ok(f) => f,
Err(e) => { Err(e) => {
@@ -237,7 +236,7 @@ async fn put_file(
}; };
let content_type = file.mime_type.clone(); let content_type = file.mime_type.clone();
let mut hasher = Sha256::new(); let mut hasher = blake3::Hasher::new();
let mut total_bytes: u64 = 0; let mut total_bytes: u64 = 0;
let mut stream = BodyStream::new(req.into_body()); let mut stream = BodyStream::new(req.into_body());
@@ -267,7 +266,7 @@ async fn put_file(
} }
drop(file_out); drop(file_out);
let hash = hex::encode(hasher.finalize()); let hash = hasher.finalize().to_hex().to_string();
// ── Atomic store: temp file → dedup blob + DB metadata update ── // ── Atomic store: temp file → dedup blob + DB metadata update ──
let result = state let result = state