perf: stream_files_in_subtree — replace Vec<File> with async Stream
Replace list_files_in_subtree (fetch_all → Vec) with stream_files_in_subtree that returns a Pin<Box<dyn Stream<Item = Result<File/FileDto>>>> backed by a PostgreSQL cursor via sqlx::fetch(). Changes: - FileReadPort::stream_files_in_subtree() returns streaming cursor (no default) - FileRetrievalUseCase::stream_files_in_subtree() maps File→FileDto on the fly - FileBlobReadRepository: async_stream::try_stream! + sqlx::fetch() cursor - batch_operations: consume stream into HashMap incrementally - zip_service: consume stream into HashMap incrementally - All stubs/mocks updated (return empty stream) Eliminates: - Double allocation: Vec<(9-tuple)> + Vec<File> materialized simultaneously - Unbounded RAM proportional to subtree size (was ~500 bytes × N files) - Latency: callers blocked until last row fetched from PG RAM is now O(folders) for the HashMap, not O(files).
This commit is contained in:
@@ -154,12 +154,15 @@ pub trait FileRetrievalUseCase: Send + Sync + 'static {
|
||||
end: Option<u64>,
|
||||
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError>;
|
||||
|
||||
/// Lists every file in the subtree rooted at `folder_id`.
|
||||
/// Streams every file in the subtree rooted at `folder_id`.
|
||||
///
|
||||
/// Default: falls back to `list_files(Some(folder_id))` (one level).
|
||||
async fn list_files_in_subtree(&self, folder_id: &str) -> Result<Vec<FileDto>, DomainError> {
|
||||
self.list_files(Some(folder_id)).await
|
||||
}
|
||||
/// Returns a streaming cursor — RAM stays O(1) per row. Callers
|
||||
/// consume incrementally (e.g. group into a HashMap by folder_id)
|
||||
/// without materializing the full result set.
|
||||
async fn stream_files_in_subtree(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<FileDto, DomainError>> + Send>>, DomainError>;
|
||||
|
||||
/// Lists files in a folder with LIMIT/OFFSET pagination.
|
||||
///
|
||||
|
||||
@@ -3,6 +3,7 @@ use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
use serde_json::Value;
|
||||
use std::path::PathBuf;
|
||||
use std::pin::Pin;
|
||||
|
||||
use crate::application::dtos::search_dto::SearchCriteriaDto;
|
||||
use crate::common::errors::DomainError;
|
||||
@@ -94,15 +95,18 @@ pub trait FileReadPort: Send + Sync + 'static {
|
||||
Ok(all.into_iter().skip(start).take(end - start).collect())
|
||||
}
|
||||
|
||||
/// Lists every file in the subtree rooted at `folder_id`.
|
||||
/// Streams every file in the subtree rooted at `folder_id`.
|
||||
///
|
||||
/// Uses an ltree `<@` join against `storage.folders` so the entire
|
||||
/// subtree is fetched in a single GiST-indexed query.
|
||||
/// subtree is resolved in a single GiST-indexed query, but rows are
|
||||
/// delivered via a PostgreSQL cursor — RAM stays O(1) per row.
|
||||
///
|
||||
/// Default: falls back to `list_files(Some(folder_id))` (one level).
|
||||
async fn list_files_in_subtree(&self, folder_id: &str) -> Result<Vec<File>, DomainError> {
|
||||
self.list_files(Some(folder_id)).await
|
||||
}
|
||||
/// Callers consume the stream incrementally (e.g. build a HashMap
|
||||
/// keyed by folder_id) without ever materializing the full Vec.
|
||||
async fn stream_files_in_subtree(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<File, DomainError>> + Send>>, DomainError>;
|
||||
|
||||
/// Search files with pagination and filtering at database level.
|
||||
///
|
||||
|
||||
@@ -792,29 +792,34 @@ impl BatchOperationService {
|
||||
|
||||
/// Adds an entire folder subtree to the ZIP using 2 bulk SQL queries
|
||||
/// (ltree `<@`) instead of N+1 per-folder traversal.
|
||||
///
|
||||
/// Files are streamed from the DB cursor — RAM for the file list is
|
||||
/// proportional to the number of *folders* (HashMap keys), not files.
|
||||
async fn add_folder_subtree_to_zip(
|
||||
&self,
|
||||
zip: &mut ZipFileWriter<tokio_util::compat::Compat<BufWriter<tokio::fs::File>>>,
|
||||
folder_id: &str,
|
||||
root_folder: &FolderDto,
|
||||
) -> Result<(), BatchOperationError> {
|
||||
// Bulk-fetch entire subtree (2 queries total)
|
||||
// Bulk-fetch folder tree (small — one entry per folder)
|
||||
let all_folders = self
|
||||
.folder_service
|
||||
.list_subtree_folders(folder_id)
|
||||
.await
|
||||
.map_err(BatchOperationError::Domain)?;
|
||||
|
||||
let all_files = self
|
||||
// Stream files from DB cursor — O(1) per row
|
||||
let mut file_stream = self
|
||||
.file_retrieval
|
||||
.list_files_in_subtree(folder_id)
|
||||
.stream_files_in_subtree(folder_id)
|
||||
.await
|
||||
.map_err(BatchOperationError::Domain)?;
|
||||
|
||||
// Group files by folder_id
|
||||
// Group files by folder_id incrementally from the stream
|
||||
let mut files_by_folder: HashMap<String, Vec<FileDto>> =
|
||||
HashMap::with_capacity(all_folders.len());
|
||||
for file in all_files {
|
||||
while let Some(file) = file_stream.next().await {
|
||||
let file = file.map_err(BatchOperationError::Domain)?;
|
||||
let fid = file.folder_id.clone().unwrap_or_default();
|
||||
files_by_folder.entry(fid).or_default().push(file);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use async_trait::async_trait;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures::{Stream, StreamExt};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::application::dtos::file_dto::FileDto;
|
||||
@@ -308,9 +309,13 @@ impl FileRetrievalUseCase for FileRetrievalService {
|
||||
self.file_read.get_file_range_stream(id, start, end).await
|
||||
}
|
||||
|
||||
async fn list_files_in_subtree(&self, folder_id: &str) -> Result<Vec<FileDto>, DomainError> {
|
||||
let files = self.file_read.list_files_in_subtree(folder_id).await?;
|
||||
Ok(files.into_iter().map(FileDto::from).collect())
|
||||
async fn stream_files_in_subtree(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<FileDto, DomainError>> + Send>>, DomainError> {
|
||||
let inner = self.file_read.stream_files_in_subtree(folder_id).await?;
|
||||
let mapped = inner.map(|r| r.map(FileDto::from));
|
||||
Ok(Box::pin(mapped))
|
||||
}
|
||||
|
||||
async fn list_files_batch(
|
||||
|
||||
@@ -505,6 +505,16 @@ mod tests {
|
||||
) -> Result<usize, DomainError> {
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
async fn stream_files_in_subtree(
|
||||
&self,
|
||||
_folder_id: &str,
|
||||
) -> Result<
|
||||
std::pin::Pin<Box<dyn futures::Stream<Item = Result<crate::domain::entities::file::File, DomainError>> + Send>>,
|
||||
DomainError,
|
||||
> {
|
||||
Ok(Box::pin(futures::stream::empty()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -4,6 +4,7 @@ use chrono::Utc;
|
||||
use futures::Stream;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -205,6 +206,13 @@ impl FileReadPort for MockFileRepository {
|
||||
) -> std::result::Result<usize, DomainError> {
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
async fn stream_files_in_subtree(
|
||||
&self,
|
||||
_folder_id: &str,
|
||||
) -> std::result::Result<Pin<Box<dyn Stream<Item = std::result::Result<File, DomainError>> + Send>>, DomainError> {
|
||||
Ok(Box::pin(futures::stream::empty()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
||||
Reference in New Issue
Block a user