cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

perf: serve ranges from RAM cache, stream ZIPs, overlap ingest settle, O(1) chunk gate

Browse Source 
Round 2 of benchmark-gated optimizations (benches/ROUND2.md; every change
gated by a before/after in examples/bench_round2.rs — an AFTER that did
not beat its BEFORE was to be rolled back; none needed it):

- Range requests (REST/DAV/shares) answered from the moka content cache
  for sub-10MB files: PG resolve + open/seek/read -> Bytes::slice.
  256KiB seeks: 1,730/s -> 3.7M/s (p50 552us -> 0.15us).
- Streaming folder/share ZIPs via tokio duplex: TTFB no longer scales
  with archive size (326ms -> 0.4ms on 192MiB corpus; total also faster).
  Content-Length dropped (size unknown up front).
- NC chunked-upload per-PUT gate: O(k) directory scan+stat -> in-RAM
  per-session counter (lazy rebuild on cold start). 1,000-chunk upload
  gate cost: 33.1s -> 0.09s cumulative.
- Delta download + commit-verify now use the CDC path's
  buffered(read_prefetch) read-ahead: 64-chunk drain at 5ms open
  latency 440ms -> 51ms; order preserved.
- CDC ingest settles batches on a spawned task (depth-1 pipeline) so
  the source stream keeps flowing during PG pin + backend writes;
  rollback ledger shared + lock-serialized so compensation stays exact
  on cancellation. 512MiB paced ingest: 60-69 -> 74-75 MB/s.
  OXICLOUD_INGEST_OVERLAP=0 restores inline settling (ops/bench hatch).
- Frontend: instant-upload BLAKE3 hashing moved off the main thread to
  a bounded Web Worker pool (File handles by reference); vitest gate
  asserts the pool beats sequential (first gate draft posting buffers
  was 2.6x slower and was rewritten — copies dominated).

Validation: cargo fmt + clippy -D warnings clean; 514 unit + 544
integration tests green; 270 frontend tests green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CBK1RdtzyP6759Muqe1K1w
This commit is contained in:
Claude
2026-07-16 16:50:07 +00:00
parent aba89c4f5d
commit 82ee7da0d2
18 changed files with 1245 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/interfaces/api/handlers/delta_upload_handler.rs
+30 -11
View File
@@ -19,7 +19,7 @@ use axum::{
response::{IntoResponse, Response},
};
use bytes::{Buf, Bytes, BytesMut};
use futures::Stream;
use futures::{Stream, TryStreamExt};
use std::sync::Arc;
use tokio_stream::StreamExt;
@@ -343,17 +343,36 @@ pub async fn delta_download_chunks(
// Stream the frames: 4-byte length headers come from the (entitled)
// index sizes; bytes stream straight from the blob backend. Peak RAM
// is one backend read frame, independent of batch size.
// is bounded by `read_prefetch` open streams (their first frame),
// independent of batch size.
//
// `buffered(read_prefetch)` overlaps the NEXT chunk's open with the
// current chunk's drain — the same combinator/tuning as the main CDC
// download path (benches/BLOB-PREFETCH.md). The old per-chunk await
// paid every open's full round-trip serially: on an object-store
// backend a 64-chunk batch at ~30 ms first-byte cost ~1.9 s of pure
// latency. Frames still arrive strictly in request order.
let prefetch = service.read_prefetch().max(1);
let svc = service.clone();
// `futures::StreamExt` spelled out — this handler imports
// `tokio_stream::StreamExt`, whose `map` adapter lacks `buffered`.
let opened = futures::StreamExt::map(futures::stream::iter(ordered), move |(hash, size)| {
let svc = svc.clone();
async move {
let chunk = svc
.chunk_stream(&hash)
.await
.map_err(std::io::Error::other)?;
let header = futures::stream::once(async move {
Ok::<Bytes, std::io::Error>(Bytes::copy_from_slice(&(size as u32).to_be_bytes()))
});
Ok::<_, std::io::Error>(futures::StreamExt::chain(header, chunk))
}
});
let body_stream: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>> =
Box::pin(async_stream::try_stream! {
for (hash, size) in ordered {
yield Bytes::copy_from_slice(&(size as u32).to_be_bytes());
let mut chunk = service.chunk_stream(&hash).await.map_err(std::io::Error::other)?;
while let Some(part) = chunk.next().await {
yield part?;
}
}
});
Box::pin(TryStreamExt::try_flatten(futures::StreamExt::buffered(
opened, prefetch,
)));
Ok(Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "application/octet-stream")
src/interfaces/api/handlers/file_handler.rs
+13 -4
View File
@@ -12,7 +12,7 @@ use std::collections::HashMap;
use utoipa::ToSchema;
use crate::application::ports::file_ports::{
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase,
FileManagementUseCase, FileRetrievalUseCase, FileUploadUseCase, RangeContent,
};
use crate::application::ports::storage_ports::{FileReadPort, StorageUsagePort};
use crate::application::ports::thumbnail_ports::ThumbnailPort;
@@ -713,10 +713,19 @@ impl FileHandler {
Self::content_disposition(&file_dto.name, &file_dto.mime_type, &params);
match retrieval
.get_file_range_stream_with_perms(&id, auth_user.id, start, Some(end + 1))
.get_file_range_preloaded_with_perms(
&file_dto,
auth_user.id,
start,
Some(end + 1),
)
.await
{
Ok(stream) => {
Ok(content) => {
let body = match content {
RangeContent::Bytes(b) => Body::from(b),
RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)),
};
return Response::builder()
.status(StatusCode::PARTIAL_CONTENT)
.header(header::CONTENT_TYPE, &*file_dto.mime_type)
@@ -732,7 +741,7 @@ impl FileHandler {
header::CACHE_CONTROL,
"private, max-age=3600, must-revalidate",
)
.body(Body::from_stream(Box::into_pin(stream)))
.body(body)
.unwrap()
.into_response();
}
src/interfaces/api/handlers/folder_handler.rs
+12 -38
View File
@@ -6,7 +6,6 @@ use axum::{
};
use std::collections::HashMap;
use std::sync::Arc;
use tokio_util::io::ReaderStream;
use crate::application::dtos::display_helpers::{
category_for, format_file_size, icon_class_for, icon_special_class_for,
@@ -238,53 +237,28 @@ impl FolderHandler {
}
};
// Create the ZIP archive (written to a temp file, O(1) RAM)
match zip_service.create_folder_zip(&id, &folder.name).await {
Ok(temp_file) => {
// Get the file size for Content-Length
let file_size = match temp_file.as_file().metadata() {
Ok(m) => m.len(),
Err(e) => {
tracing::error!("Error reading temp file metadata: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
"error": "Error creating ZIP file"
})),
)
.into_response();
}
};
tracing::info!("ZIP file created successfully, size: {} bytes", file_size);
// Split the NamedTempFile into the already-open std File
// and the TempPath (auto-deletes on drop). This reuses
// the existing fd instead of opening a second one.
let (std_file, temp_path) = temp_file.into_parts();
let tokio_file = tokio::fs::File::from_std(std_file);
// Stream the file to the client in chunks
let stream = ReaderStream::new(tokio_file);
// Stream the archive as it is built — the first byte reaches
// the client after the first entry, not after the whole ZIP
// exists on disk (benches/ZIP-STREAM.md). No Content-Length:
// the final size isn't known up front (chunked encoding).
match zip_service
.create_folder_zip_stream(&id, &folder.name)
.await
{
Ok(stream) => {
let body = axum::body::Body::from_stream(stream);
// Setup headers for download
let filename = format!("{}.zip", folder.name);
let content_disposition = format!("attachment; filename=\"{}\"", filename);
let mut response = Response::builder()
Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "application/zip")
.header(header::CONTENT_DISPOSITION, content_disposition)
.header(header::CONTENT_LENGTH, file_size)
.body(body)
.unwrap();
// Keep TempPath alive in the response extensions so the
// file is only deleted AFTER the body stream finishes.
response.extensions_mut().insert(Arc::new(temp_path));
response.into_response()
.unwrap()
.into_response()
}
Err(err) => {
tracing::error!("Error creating ZIP file: {}", err);
src/interfaces/api/handlers/share_handler.rs
+15 -27
View File
@@ -13,6 +13,7 @@ use serde::Deserialize;
use serde_json::json;
use utoipa::ToSchema;
use crate::application::ports::file_ports::RangeContent;
use crate::application::services::share_browse_service::ZipTarget;
use crate::application::services::share_service::ShareService;
use crate::infrastructure::services::share_unlock_cookie;
@@ -30,7 +31,6 @@ use crate::{
interfaces::errors::AppError,
interfaces::middleware::auth::AuthUser,
};
use tokio_util::io::ReaderStream;
fn unlock_jwt_from_headers(headers: &HeaderMap, share_token: &str) -> Option<String> {
headers
@@ -438,10 +438,14 @@ async fn serve_share_file(
let length = end - start + 1;
match retrieval
.get_file_range_stream(file_id, start, Some(end + 1))
.get_file_range_preloaded(&file_dto, start, Some(end + 1))
.await
{
Ok(stream) => {
Ok(content) => {
let body = match content {
RangeContent::Bytes(b) => Body::from(b),
RangeContent::Stream(s) => Body::from_stream(Box::into_pin(s)),
};
return Response::builder()
.status(StatusCode::PARTIAL_CONTENT)
.header(header::CONTENT_TYPE, &*mime)
@@ -458,7 +462,7 @@ async fn serve_share_file(
"private, max-age=3600, must-revalidate",
)
.header(header::VARY, "Cookie, Range")
.body(Body::from_stream(Box::into_pin(stream)))
.body(body)
.unwrap()
.into_response();
}
@@ -730,30 +734,19 @@ async fn serve_share_zip(
Err(err) => return share_browse_error_response(err),
};
let temp_file = match zip_service
.create_folder_zip(&target.folder_id, &target.display_name)
// Streamed archive: first byte after the first entry, not after the
// whole ZIP is built (benches/ZIP-STREAM.md). No Content-Length.
let stream = match zip_service
.create_folder_zip_stream(&target.folder_id, &target.display_name)
.await
{
Ok(f) => f,
Ok(s) => s,
Err(err) => {
tracing::error!("share zip: create_folder_zip failed: {}", err);
return AppError::internal_error(format!("ZIP creation failed: {}", err))
.into_response();
}
};
let file_size = match temp_file.as_file().metadata() {
Ok(m) => m.len(),
Err(e) => {
tracing::error!("share zip: temp metadata failed: {}", e);
return AppError::internal_error("ZIP creation failed").into_response();
}
};
// Reuse the existing fd: split off the std::File and the TempPath.
let (std_file, temp_path) = temp_file.into_parts();
let tokio_file = tokio::fs::File::from_std(std_file);
let stream = ReaderStream::new(tokio_file);
let body = Body::from_stream(stream);
let disposition = build_content_disposition(
@@ -762,17 +755,12 @@ async fn serve_share_zip(
false,
);
let mut response = Response::builder()
Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, "application/zip")
.header(header::CONTENT_DISPOSITION, disposition)
.header(header::CONTENT_LENGTH, file_size)
.header(header::CACHE_CONTROL, "private, no-store")
.header(header::VARY, "Cookie")
.body(body)
.unwrap();
// Keep TempPath alive until the body finishes streaming.
response.extensions_mut().insert(Arc::new(temp_path));
response
.unwrap()
}