cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

perf: stream_files_in_subtree — replace Vec<File> with async Stream

Browse Source 
Replace list_files_in_subtree (fetch_all → Vec) with stream_files_in_subtree
that returns a Pin<Box<dyn Stream<Item = Result<File/FileDto>>>> backed by a
PostgreSQL cursor via sqlx::fetch().

Changes:
- FileReadPort::stream_files_in_subtree() returns streaming cursor (no default)
- FileRetrievalUseCase::stream_files_in_subtree() maps File→FileDto on the fly
- FileBlobReadRepository: async_stream::try_stream! + sqlx::fetch() cursor
- batch_operations: consume stream into HashMap incrementally
- zip_service: consume stream into HashMap incrementally
- All stubs/mocks updated (return empty stream)

Eliminates:
- Double allocation: Vec<(9-tuple)> + Vec<File> materialized simultaneously
- Unbounded RAM proportional to subtree size (was ~500 bytes × N files)
- Latency: callers blocked until last row fetched from PG

RAM is now O(folders) for the HashMap, not O(files).
This commit is contained in:
Dionisio
2026-02-26 00:07:10 +01:00
parent 1ad7a32a61
commit 9f8a6f5177
9 changed files with 127 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/infrastructure/repositories/pg/file_blob_read_repository.rs
+44 -38
View File
@@ -9,9 +9,10 @@
use async_trait::async_trait;
use bytes::Bytes;
use futures::Stream;
use futures::{Stream, TryStreamExt};
use moka::sync::Cache;
use sqlx::PgPool;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
@@ -437,45 +438,50 @@ impl FileReadPort for FileBlobReadRepository {
}
}
/// Lists every file in the subtree rooted at `folder_id` (inclusive).
/// Streams every file in the subtree rooted at `folder_id`.
///
/// Single GiST-indexed query via ltree `<@`.
/// Ordered by `(fo.path, fi.name)` so callers iterate in directory order.
async fn list_files_in_subtree(&self, folder_id: &str) -> Result<Vec<File>, DomainError> {
let rows: Vec<(
String,
String,
Option<String>,
Option<String>,
i64,
String,
i64,
i64,
Option<String>,
)> = sqlx::query_as(
r#"
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
fi.size, fi.mime_type,
EXTRACT(EPOCH FROM fi.created_at)::bigint,
EXTRACT(EPOCH FROM fi.updated_at)::bigint,
fi.user_id::text
FROM storage.files fi
JOIN storage.folders fo ON fo.id = fi.folder_id
WHERE fo.lpath <@ (SELECT lpath FROM storage.folders WHERE id = $1::uuid)
AND NOT fi.is_trashed
ORDER BY fo.path, fi.name
"#,
)
.bind(folder_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("FileBlobRead", format!("subtree files: {e}")))?;
/// Single GiST-indexed query via ltree `<@`. Results are delivered
/// through a PostgreSQL cursor — RAM stays O(1) per row.
async fn stream_files_in_subtree(
&self,
folder_id: &str,
) -> Result<Pin<Box<dyn Stream<Item = Result<File, DomainError>> + Send>>, DomainError> {
let pool = Arc::clone(&self.pool);
let folder_id = folder_id.to_owned();
rows.into_iter()
.map(|(id, name, fid, fpath, size, mime, ca, ma, uid)| {
Self::row_to_file(id, name, fid, fpath, size, mime, ca, ma, uid)
})
.collect()
let stream = async_stream::try_stream! {
let mut row_stream = sqlx::query_as::<_, (
String, String, Option<String>, Option<String>,
i64, String, i64, i64, Option<String>,
)>(
r#"
SELECT fi.id::text, fi.name, fi.folder_id::text, fo.path,
fi.size, fi.mime_type,
EXTRACT(EPOCH FROM fi.created_at)::bigint,
EXTRACT(EPOCH FROM fi.updated_at)::bigint,
fi.user_id::text
FROM storage.files fi
JOIN storage.folders fo ON fo.id = fi.folder_id
WHERE fo.lpath <@ (SELECT lpath FROM storage.folders WHERE id = $1::uuid)
AND NOT fi.is_trashed
ORDER BY fo.path, fi.name
"#,
)
.bind(&folder_id)
.fetch(pool.as_ref());
while let Some(row) = row_stream.try_next().await.map_err(|e| {
DomainError::internal_error("FileBlobRead", format!("subtree stream: {e}"))
})? {
let (id, name, fid, fpath, size, mime, ca, ma, uid) = row;
let file = FileBlobReadRepository::row_to_file(
id, name, fid, fpath, size, mime, ca, ma, uid,
)?;
yield file;
}
};
Ok(Box::pin(stream))
}
/// Search files with filtering and pagination at database level.
src/infrastructure/services/zip_service.rs
+15 -12
View File
@@ -94,33 +94,36 @@ impl ZipService {
}
};
// ── 1. Bulk-fetch the entire subtree (2 queries total) ───────────
// ── 1. Bulk-fetch folder tree (small — one entry per folder) ────
let all_folders = self
.folder_service
.list_subtree_folders(folder_id)
.await
.map_err(|e| ZipError::FolderContentsError(format!("subtree folders: {}", e)))?;
let all_files = self
// ── 2. Stream files from DB cursor — O(1) per row ───────────────
let mut file_stream = self
.file_service
.list_files_in_subtree(folder_id)
.stream_files_in_subtree(folder_id)
.await
.map_err(|e| ZipError::FolderContentsError(format!("subtree files: {}", e)))?;
// Group files by folder_id incrementally from the stream
let mut files_by_folder: HashMap<String, Vec<FileDto>> =
HashMap::with_capacity(all_folders.len());
while let Some(file) = file_stream.next().await {
let file =
file.map_err(|e| ZipError::FolderContentsError(format!("subtree file: {}", e)))?;
let fid = file.folder_id.clone().unwrap_or_default();
files_by_folder.entry(fid).or_default().push(file);
}
info!(
"ZIP subtree: {} folders, {} files",
all_folders.len(),
all_files.len()
files_by_folder.values().map(|v| v.len()).sum::<usize>()
);
// ── 2. Group files by folder_id ──────────────────────────────────
let mut files_by_folder: HashMap<String, Vec<FileDto>> =
HashMap::with_capacity(all_folders.len());
for file in all_files {
let fid = file.folder_id.clone().unwrap_or_default();
files_by_folder.entry(fid).or_default().push(file);
}
// ── 3. Build a mapping: folder_id → ZIP-relative path ────────────
//
// The root folder's DB path is e.g. "/users/alice/Documents".