cjw/Oxicloud
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge upstream/main into feat/external-file-mounts

Browse Source 
Resolve conflicts between the external-file-mounts feature and upstream's
D5/D7 refactor (per-file provenance, keyset pagination, cross-drive move
gates, resource-access hook, folder-cascade lifecycle hook).

Key resolutions:
- FolderService::new now takes (repo, authz, file_lifecycle, mount_router);
  all callers + DI updated.
- FileRetrievalService / FileManagementService keep both the mount_router
  and the new resource_access_hook / drive_repo / storage_usage wiring.
- list_files_batch_with_perms: adapt the mount branch from offset- to
  keyset (after_name) pagination, mirroring paginate_mount_entries.
- download_file_impl: keep upstream's &HeaderMap + `impl IntoResponse + use<>`
  signature, retain the mount-download branch.
- Mount DTOs: the retired `owner_id` field maps onto created_by/updated_by
  (the mount owner) — the fields the frontend now uses for owner display.
- admin/+page.svelte: keep upstream's user-delete modal + the 'mounts' tab.
- Bump memmap2 0.9.10 -> 0.9.11 (RUSTSEC critical advisory fix) and
  regenerate Cargo.lock against the merged Cargo.toml.
This commit is contained in:
Bradley Nelson
2026-07-21 17:09:36 -06:00
parent 94e5aec4af 67fe944c2a
commit 9f4bc54113
600 changed files with 105575 additions and 14440 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/application/services/file_retrieval_service.rs
+246 -60
View File
@@ -7,7 +7,10 @@ use crate::application::dtos::file_dto::FileDto;
use crate::application::ports::authorization_ports::AuthorizationEngine;
use crate::application::ports::blob_storage_ports::BlobStream;
use crate::application::ports::external_mount_ports::MountStat;
use crate::application::ports::file_ports::{FileRetrievalUseCase, OptimizedFileContent};
use crate::application::ports::file_ports::{
FileRetrievalUseCase, OptimizedFileContent, RangeContent,
};
use crate::application::ports::resource_access_hook::ResourceAccessHook;
use crate::application::ports::storage_ports::FileReadPort;
use crate::application::services::mount_registry::MountConfig;
use crate::common::errors::DomainError;
@@ -40,6 +43,11 @@ pub struct FileRetrievalService {
/// External-mount classifier for path-based resolution (WebDAV/NextCloud).
/// `None` in the simple/test constructor → no mount support.
mount_router: Option<Arc<crate::application::services::external_mount_router::MountRouter>>,
/// Optional read-event observer. Currently fans out to the Recent-list
/// recorder; future observers (audit trail, "last seen by", …) attach
/// to the same hook so service code only knows the trait, not the impl.
/// `None` for the test/stub path that constructs via [`Self::new`].
resource_access_hook: Option<Arc<dyn ResourceAccessHook>>,
}
impl FileRetrievalService {
@@ -53,6 +61,7 @@ impl FileRetrievalService {
transcode: None,
authz: None,
mount_router: None,
resource_access_hook: None,
}
}
@@ -70,6 +79,7 @@ impl FileRetrievalService {
transcode: Some(transcode),
authz: Some(authz),
mount_router: None,
resource_access_hook: None,
}
}
@@ -83,6 +93,14 @@ impl FileRetrievalService {
self
}
/// Builder: attach a [`ResourceAccessHook`] that fires after every
/// authorised `_with_perms` read. Without it the service is silent —
/// existing behaviour for stub / test paths.
pub fn with_resource_access_hook(mut self, hook: Arc<dyn ResourceAccessHook>) -> Self {
self.resource_access_hook = Some(hook);
self
}
/// Test-only constructor: authorization engine without the cache/transcode
/// tiers. The external-mount read methods only consult `authz` + the
/// provider, so this is sufficient to exercise their authorization.
@@ -97,6 +115,24 @@ impl FileRetrievalService {
transcode: None,
authz: Some(authz),
mount_router: None,
resource_access_hook: None,
}
}
/// Fire the access hook if registered. Called from every `_with_perms`
/// read after the authZ + lookup has succeeded (never on failure
/// paths — denied reads must not surface in Recent).
///
/// `pub` because the WebDAV / NextCloud DAV handlers resolve files
/// by path and authorise via that resolver, not via the
/// `*_with_perms` service methods — they then serve content through
/// the no-perms `get_file_stream` / `get_file_range_stream`. Those
/// handlers must call this directly after their own authZ has
/// passed so cross-protocol downloads (NC desktop, davx5, native
/// `/webdav/`) also surface in Recent.
pub fn notify_file_accessed(&self, caller_id: Uuid, file_id: &str) {
if let Some(hook) = &self.resource_access_hook {
hook.on_file_accessed(caller_id, file_id);
}
}
@@ -112,7 +148,26 @@ impl FileRetrievalService {
) -> Result<Bytes, DomainError> {
let stream = file_read.get_file_stream(id).await?;
let mut stream = Pin::from(stream);
let mut buf = BytesMut::with_capacity(capacity);
// Most sub-threshold reads arrive as ONE owned contiguous frame from the
// backend (the local ReaderStream emits ≤256 KiB frames, and a
// sub-threshold blob fits in one). Return that frame directly instead of
// copying the whole payload a second time into a fresh BytesMut; only a
// multi-frame read pays the pre-sized concat — byte-identical output
// (benches/ROUND29.md §C).
let Some(first) = stream.next().await else {
return Ok(Bytes::new());
};
let first = first.map_err(|e| {
DomainError::internal_error("File", format!("Stream read error: {}", e))
})?;
let Some(second) = stream.next().await else {
return Ok(first);
};
let mut buf = BytesMut::with_capacity(capacity.max(first.len()));
buf.extend_from_slice(&first);
buf.extend_from_slice(&second.map_err(|e| {
DomainError::internal_error("File", format!("Stream read error: {}", e))
})?);
while let Some(chunk) = stream.next().await {
buf.extend_from_slice(&chunk.map_err(|e| {
DomainError::internal_error("File", format!("Stream read error: {}", e))
@@ -263,7 +318,6 @@ impl FileRetrievalService {
) -> Result<(FileDto, OptimizedFileContent), DomainError> {
let mime_type = dto.mime_type.clone();
let file_size = dto.size;
let file_name = dto.name.clone();
// The content cache is content-addressed: keyed by the blob hash, not
// the file id. Identical content deduplicated to one blob on disk is
// then cached ONCE in RAM and shared by every file/user that references
@@ -271,34 +325,39 @@ impl FileRetrievalService {
// construction, so entries never go stale (no invalidation needed). A
// stub DTO without a hash disables caching for that request rather than
// colliding every hash-less file on the key "".
let cache_key = dto.content_hash.clone();
let cacheable = !cache_key.is_empty();
let cacheable = !dto.content_hash.is_empty();
let do_transcode = accept_webp && !prefer_original;
// ── Tier 1: Hot cache + transcode (<10 MB) ──────────
if file_size < CACHE_THRESHOLD {
// Fetch the raw blob bytes. When cacheable, `get_or_load` serves
// from the content cache on a hit and, on a miss, coalesces every
// concurrent request for the same blob hash into a SINGLE disk read
// (single-flight) — no thundering herd under load. Hash-less stub
// DTOs are uncacheable and stream straight from disk.
// Probe the content cache with a BORROW first: a hit serves the blob
// straight from RAM, and only a miss builds the owned load arguments
// (the quoted-etag / key / id Strings) that a hit would otherwise
// allocate and immediately discard (benches/ROUND29.md §B). On a miss
// `load_and_cache` still coalesces concurrent requests for the same
// blob hash into a SINGLE disk read (single-flight) — no thundering
// herd. Hash-less stub DTOs are uncacheable and stream from disk.
let content_bytes = if cacheable && let Some(cache) = &self.content_cache {
let etag: Arc<str> = format!("\"{}\"", cache_key).into();
let ct: Arc<str> = mime_type.clone();
let file_read = Arc::clone(&self.file_read);
let id_owned = id.to_string();
let cap = file_size as usize;
let (bytes, _etag, _ct) = cache
.get_or_load(cache_key.clone(), etag, ct, async move {
debug!("💾 TIER 1 Cache MISS: {} – loading from disk", id_owned);
Self::read_full(&file_read, &id_owned, cap).await
})
.await?;
bytes
if let Some((bytes, ..)) = cache.get(&dto.content_hash).await {
bytes
} else {
let etag: Arc<str> = format!("\"{}\"", dto.content_hash).into();
let ct: Arc<str> = mime_type.clone();
let file_read = Arc::clone(&self.file_read);
let id_owned = id.to_string();
let cap = file_size as usize;
let (bytes, ..) = cache
.load_and_cache(dto.content_hash.to_string(), etag, ct, async move {
debug!("💾 TIER 1 Cache MISS: {} – loading from disk", id_owned);
Self::read_full(&file_read, &id_owned, cap).await
})
.await?;
bytes
}
} else {
debug!(
"💾 TIER 1 (uncacheable): {} – streaming from disk",
file_name
dto.name
);
Self::read_full(&self.file_read, id, file_size as usize).await?
};
@@ -330,7 +389,7 @@ impl FileRetrievalService {
// ── Tier 2 + 3: Streaming (≥10 MB) ──────────────────
info!(
"📡 TIER 2 STREAMING: {} ({} MB)",
file_name,
dto.name,
file_size / (1024 * 1024)
);
let stream = self.file_read.get_file_stream(id).await?;
@@ -347,6 +406,109 @@ impl FileRetrievalService {
let files = self.file_read.get_files_by_ids(ids).await?;
Ok(files.into_iter().map(FileDto::from).collect())
}
/// Batched, authorized multi-get for the ZIP-download multi-select — the
/// batch form of [`FileRetrievalUseCase::get_file_with_perms`] over an
/// explicit id list.
///
/// Authorizes `Read` on every id in ONE `check_files_read_batch`
/// round-trip (which resolves all drives in a single query AND primes the
/// resource→drive cache, so the per-file re-check the subsequent stream
/// open performs becomes a cache hit), then fetches only the authorized ids
/// in ONE `get_files_by_ids` query. Replaces `download_zip`'s per-file
/// `require_file` + `get_file` loop — 2 round-trips/file → 2 total.
///
/// Returns the authorized, existing files; a denied / missing / unparseable
/// id is simply **absent** from the result (the caller re-associates by id
/// and skips the rest, exactly as the per-file loop skipped a denied /
/// missing `get_file_with_perms`). Read-authorization is identical to the
/// per-file path (`check_files_read_batch` is documented and gated as
/// semantically identical to looping `require`). Recents recording is left
/// to the subsequent per-file stream open (`get_file_stream_with_perms`),
/// which records it (throttle-coalesced) — same net effect as the old
/// loop's `notify_file_accessed` + stream double-notify. Fail-closed if no
/// engine was injected, mirroring [`Self::require_file`].
pub async fn get_files_by_ids_with_perms(
&self,
ids: &[String],
caller_id: Uuid,
) -> Result<Vec<FileDto>, DomainError> {
let authz = self.authz.as_ref().ok_or_else(|| {
DomainError::internal_error("FileRetrieval", "Authorization engine unavailable")
})?;
// Unparseable ids can't be authorized (the per-file path 404s on them),
// so drop them here — they stay absent from the authorized set.
let uuids: Vec<Uuid> = ids.iter().filter_map(|s| Uuid::parse_str(s).ok()).collect();
if uuids.is_empty() {
return Ok(Vec::new());
}
let allowed = authz
.check_files_read_batch(Subject::User(caller_id), &uuids)
.await?;
if allowed.is_empty() {
return Ok(Vec::new());
}
let allowed_ids: Vec<String> = allowed.iter().map(Uuid::to_string).collect();
let files = self.file_read.get_files_by_ids(&allowed_ids).await?;
Ok(files.into_iter().map(FileDto::from).collect())
}
/// Range read for HTTP Range Requests, cache-aware.
///
/// Media players and PDF viewers fetch these files *exclusively* through
/// Range requests (a `bytes=0-` probe, then seeks) — the plain streaming
/// path paid 1 PG round-trip (blob-hash resolve) + a chunk open/seek for
/// EVERY seek, even when the whole blob was already sitting in the moka
/// content cache as one contiguous `Bytes`. For sub-`CACHE_THRESHOLD`
/// files this now answers from the cache: `Bytes::slice` is a refcount
/// bump — zero copy, zero I/O, zero PG (benches/RANGE-CACHE.md). A miss
/// populates the cache via the same single-flight `get_or_load` Tier 1
/// uses, so one probe warms every subsequent seek. `end` is exclusive
/// (callers pass `Some(last_byte + 1)`), matching the streaming variant.
pub async fn get_file_range_preloaded(
&self,
dto: &FileDto,
start: u64,
end: Option<u64>,
) -> Result<RangeContent, DomainError> {
let cacheable = dto.size < CACHE_THRESHOLD && !dto.content_hash.is_empty();
if cacheable && let Some(cache) = &self.content_cache {
// Probe with a BORROW first: the video-scrub steady state is a cache
// hit, and a hit must not allocate the owned load args (quoted-etag /
// key / id Strings) it would immediately discard — those are built
// only on the miss branch (benches/ROUND29.md §B). A miss still
// populates via the same single-flight coalescing.
let bytes = if let Some((bytes, ..)) = cache.get(&dto.content_hash).await {
bytes
} else {
let etag: Arc<str> = format!("\"{}\"", dto.content_hash).into();
let ct: Arc<str> = dto.mime_type.clone();
let file_read = Arc::clone(&self.file_read);
let id_owned = dto.id.clone();
let cap = dto.size as usize;
let (bytes, ..) = cache
.load_and_cache(dto.content_hash.to_string(), etag, ct, async move {
debug!("💾 Range cache MISS: {} – loading from disk", id_owned);
Self::read_full(&file_read, &id_owned, cap).await
})
.await?;
bytes
};
let len = bytes.len() as u64;
let s = start.min(len) as usize;
let e = end.unwrap_or(len).min(len) as usize;
if s <= e {
return Ok(RangeContent::Bytes(bytes.slice(s..e)));
}
// Degenerate range the validator should have rejected — fall
// through to the streaming path rather than panic on slice.
}
let stream = self
.file_read
.get_file_range_stream(&dto.id, start, end)
.await?;
Ok(RangeContent::Stream(stream))
}
}
impl FileRetrievalUseCase for FileRetrievalService {
@@ -358,6 +520,11 @@ impl FileRetrievalUseCase for FileRetrievalService {
async fn get_file_with_perms(&self, id: &str, caller_id: Uuid) -> Result<FileDto, DomainError> {
self.require_file(id, Permission::Read, caller_id).await?;
let file = self.file_read.get_file(id).await?;
// After authZ + lookup succeed: this caller has just inspected the
// file. Recent listing observes via the hook. The throttle in the
// recording impl coalesces repeat metadata fetches against the same
// file (file viewer poll, browse-then-download pattern).
self.notify_file_accessed(caller_id, id);
Ok(FileDto::from(file))
}
@@ -421,19 +588,17 @@ impl FileRetrievalUseCase for FileRetrievalService {
folder_id: Option<&str>,
owner_id: Uuid,
) -> Result<Vec<FileDto>, DomainError> {
if folder_id.is_some() {
// folder id is defined, check permissions
self.require_target_folder_perm(folder_id, Permission::Read, owner_id)
.await?;
self.list_files(folder_id).await
} else {
// no folder id, get owners's files' root
let files = self
.file_read
.list_files_for_owner(folder_id, owner_id)
.await?;
Ok(files.into_iter().map(FileDto::from).collect())
// Files always have a `folder_id` in the D0+ model — there is no
// longer any concept of "root-level files". A `None` from the
// caller means the query string was missing `folder_id`; reject
// with a clear error rather than returning an empty set from a
// meaningless root-level query.
if folder_id.is_none() {
return Err(DomainError::validation_error("folder_id is required"));
}
self.require_target_folder_perm(folder_id, Permission::Read, owner_id)
.await?;
self.list_files(folder_id).await
}
async fn get_file_stream(
@@ -454,6 +619,7 @@ impl FileRetrievalUseCase for FileRetrievalService {
caller_id: Uuid,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
self.require_file(id, Permission::Read, caller_id).await?;
self.notify_file_accessed(caller_id, id);
self.file_read.get_file_stream(id).await
}
@@ -480,6 +646,7 @@ impl FileRetrievalUseCase for FileRetrievalService {
self.require_file(id, Permission::Read, caller_id).await?;
let file = self.file_read.get_file(id).await?;
let dto = FileDto::from(file);
self.notify_file_accessed(caller_id, id);
self.optimized_inner(id, dto, accept_webp, prefer_original)
.await
}
@@ -521,6 +688,10 @@ impl FileRetrievalUseCase for FileRetrievalService {
end: Option<u64>,
) -> Result<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>, DomainError> {
self.require_file(id, Permission::Read, caller_id).await?;
// Range requests are bursty (video seeks, NC chunked downloads) —
// the recording hook's per-(caller, file) throttle absorbs the
// storm so one watched video lands as one Recent row, not 1000.
self.notify_file_accessed(caller_id, id);
self.file_read.get_file_range_stream(id, start, end).await
}
@@ -537,12 +708,12 @@ impl FileRetrievalUseCase for FileRetrievalService {
async fn list_files_batch(
&self,
folder_id: Option<&str>,
offset: i64,
after_name: Option<&str>,
limit: i64,
) -> Result<Vec<FileDto>, DomainError> {
let files = self
.file_read
.list_files_batch(folder_id, offset, limit)
.list_files_batch(folder_id, after_name, limit)
.await?;
Ok(files.into_iter().map(FileDto::from).collect())
}
@@ -551,21 +722,21 @@ impl FileRetrievalUseCase for FileRetrievalService {
&self,
folder_id: Option<&str>,
owner_id: Uuid,
offset: i64,
after_name: Option<&str>,
limit: i64,
) -> Result<Vec<FileDto>, DomainError> {
// External mount: list files from the provider (WebDAV/NextCloud
// PROPFIND Depth:1 file loop). Authz collapses on the mount root.
// Keyset pagination by name mirrors `paginate_mount_entries` —
// provider order isn't guaranteed, so sort before slicing on
// `after_name`.
if let Some(fid) = folder_id
&& let Some(router) = &self.mount_router
{
use crate::application::services::external_mount_router::ResolvedId;
let resolved = match router.classify(fid) {
ResolvedId::Regular => None,
ResolvedId::MountRoot { cfg } => Some((
cfg,
crate::domain::services::external_mount_id::NodeId::default(),
)),
ResolvedId::MountRoot { cfg } => Some((cfg, NodeId::default())),
ResolvedId::MountChild { cfg, node_id } => Some((cfg, node_id)),
};
if let Some((cfg, node)) = resolved {
@@ -578,34 +749,49 @@ impl FileRetrievalUseCase for FileRetrievalService {
)
.await?;
}
let entries = cfg.provider.list_dir(&node).await?;
let files: Vec<FileDto> = entries
.iter()
let mut entries: Vec<_> = cfg
.provider
.list_dir(&node)
.await?
.into_iter()
.filter(|e| !e.is_dir)
.skip(offset.max(0) as usize)
.collect();
entries.sort_by_key(|e| e.name.to_lowercase());
let start = match after_name {
Some(name) => entries
.iter()
.position(|e| name.eq_ignore_ascii_case(&e.name))
.map(|i| i + 1)
.unwrap_or(0),
None => 0,
};
let files: Vec<FileDto> = entries
.into_iter()
.skip(start)
.take(limit.max(0) as usize)
.map(|e| {
crate::application::services::mount_dto::mount_entry_file_dto(&cfg, fid, e)
crate::application::services::mount_dto::mount_entry_file_dto(&cfg, fid, &e)
})
.collect();
return Ok(files);
}
}
if folder_id.is_some() {
// folder id is defined, check permissions
self.require_target_folder_perm(folder_id, Permission::Read, owner_id)
.await?;
let files = self
.file_read
.list_files_batch(folder_id, offset, limit)
.await?;
return Ok(files.into_iter().map(FileDto::from).collect());
}
// Post-D0: every file lives in a folder — `storage.files.folder_id`
// is NOT NULL. `folder_id = None` means the caller is asking for
// "root-level files", which by design return an empty set: the
// WebDAV synthetic root only lists drive-root folders as
// children. Skip the DB round-trip and the pre-D7 owner-fallback
// query (which used to hit `_for_owner` and would have driven
// the `files.user_id` filter this refactor is retiring).
let Some(_) = folder_id else {
return Ok(Vec::new());
};
self.require_target_folder_perm(folder_id, Permission::Read, owner_id)
.await?;
let files = self
.file_read
.list_files_batch_for_owner(folder_id, owner_id, offset, limit)
.list_files_batch(folder_id, after_name, limit)
.await?;
Ok(files.into_iter().map(FileDto::from).collect())
}