Merge upstream/main into feat/external-file-mounts

Resolve conflicts between the external-file-mounts feature and upstream's
D5/D7 refactor (per-file provenance, keyset pagination, cross-drive move
gates, resource-access hook, folder-cascade lifecycle hook).

Key resolutions:
- FolderService::new now takes (repo, authz, file_lifecycle, mount_router);
  all callers + DI updated.
- FileRetrievalService / FileManagementService keep both the mount_router
  and the new resource_access_hook / drive_repo / storage_usage wiring.
- list_files_batch_with_perms: adapt the mount branch from offset- to
  keyset (after_name) pagination, mirroring paginate_mount_entries.
- download_file_impl: keep upstream's &HeaderMap + `impl IntoResponse + use<>`
  signature, retain the mount-download branch.
- Mount DTOs: the retired `owner_id` field maps onto created_by/updated_by
  (the mount owner) — the fields the frontend now uses for owner display.
- admin/+page.svelte: keep upstream's user-delete modal + the 'mounts' tab.
- Bump memmap2 0.9.10 -> 0.9.11 (RUSTSEC critical advisory fix) and
  regenerate Cargo.lock against the merged Cargo.toml.
This commit is contained in:
Bradley Nelson
2026-07-21 17:09:36 -06:00
600 changed files with 105575 additions and 14440 deletions
@@ -13,7 +13,7 @@ use crate::application::dtos::calendar_dto::{
CalendarDto, CalendarEventDto, CreateCalendarDto, CreateEventDto, CreateEventICalDto,
UpdateCalendarDto, UpdateEventDto,
};
use crate::application::ports::calendar_ports::CalendarStoragePort;
use crate::application::ports::calendar_ports::{CalendarStoragePort, UpsertEventsResult};
use crate::common::errors::{DomainError, ErrorKind};
use crate::domain::entities::calendar::Calendar;
use crate::domain::entities::calendar_event::CalendarEvent;
@@ -39,6 +39,14 @@ impl CalendarStorageAdapter {
event_repository,
}
}
/// Delegates to [`CalendarPgRepository::has_owned_calendar`] — the
/// `EXISTS` short-circuit used by the login provisioning hook instead
/// of hydrating every owned calendar to test emptiness
/// (benches/ROUND13.md §Q2).
pub async fn has_owned_calendar(&self, owner_id: Uuid) -> Result<bool, DomainError> {
self.calendar_repository.has_owned_calendar(owner_id).await
}
}
impl CalendarStoragePort for CalendarStorageAdapter {
@@ -115,6 +123,11 @@ impl CalendarStoragePort for CalendarStorageAdapter {
Ok(CalendarDto::from(calendar))
}
async fn get_calendars_by_ids(&self, ids: &[Uuid]) -> Result<Vec<CalendarDto>, DomainError> {
let calendars = self.calendar_repository.find_calendars_by_ids(ids).await?;
Ok(calendars.into_iter().map(CalendarDto::from).collect())
}
async fn list_calendars_by_owner(
&self,
owner_id: Uuid,
@@ -126,17 +139,6 @@ impl CalendarStoragePort for CalendarStorageAdapter {
Ok(calendars.into_iter().map(CalendarDto::from).collect())
}
async fn list_calendars_shared_with_user(
&self,
user_id: Uuid,
) -> Result<Vec<CalendarDto>, DomainError> {
let calendars = self
.calendar_repository
.list_calendars_shared_with_user(user_id)
.await?;
Ok(calendars.into_iter().map(CalendarDto::from).collect())
}
async fn list_public_calendars(
&self,
limit: i64,
@@ -149,78 +151,6 @@ impl CalendarStoragePort for CalendarStorageAdapter {
Ok(calendars.into_iter().map(CalendarDto::from).collect())
}
async fn check_calendar_access(
&self,
calendar_id: &str,
user_id: Uuid,
) -> Result<bool, DomainError> {
let uuid = Uuid::parse_str(calendar_id).map_err(|_| {
DomainError::new(
ErrorKind::InvalidInput,
"Calendar",
"Invalid calendar ID format",
)
})?;
self.calendar_repository
.user_has_calendar_access(&uuid, user_id)
.await
}
// Calendar sharing
async fn share_calendar(
&self,
calendar_id: &str,
user_id: Uuid,
access_level: &str,
) -> Result<(), DomainError> {
let uuid = Uuid::parse_str(calendar_id).map_err(|_| {
DomainError::new(
ErrorKind::InvalidInput,
"Calendar",
"Invalid calendar ID format",
)
})?;
self.calendar_repository
.share_calendar(&uuid, user_id, access_level)
.await
}
async fn remove_calendar_sharing(
&self,
calendar_id: &str,
user_id: Uuid,
) -> Result<(), DomainError> {
let uuid = Uuid::parse_str(calendar_id).map_err(|_| {
DomainError::new(
ErrorKind::InvalidInput,
"Calendar",
"Invalid calendar ID format",
)
})?;
self.calendar_repository
.remove_calendar_sharing(&uuid, user_id)
.await
}
async fn get_calendar_shares(
&self,
calendar_id: &str,
) -> Result<Vec<(String, String)>, DomainError> {
let uuid = Uuid::parse_str(calendar_id).map_err(|_| {
DomainError::new(
ErrorKind::InvalidInput,
"Calendar",
"Invalid calendar ID format",
)
})?;
self.calendar_repository.get_calendar_shares(&uuid).await
}
// Calendar properties
async fn set_calendar_property(
@@ -345,6 +275,73 @@ impl CalendarStoragePort for CalendarStorageAdapter {
Ok(CalendarEventDto::from(created))
}
async fn upsert_ical_events(
&self,
dto: CreateEventICalDto,
) -> Result<UpsertEventsResult, DomainError> {
let calendar_id = Uuid::parse_str(&dto.calendar_id).map_err(|_| {
DomainError::new(
ErrorKind::InvalidInput,
"Event",
"Invalid calendar ID format",
)
})?;
// Verify calendar exists before touching the events table.
let _calendar = self
.calendar_repository
.find_calendar_by_id(&calendar_id)
.await?;
// Split the body into one CalendarEvent per VEVENT. A body
// with zero VEVENTs (or only VTODOs / VJOURNALs) returns
// InvalidInput here — which the handler layer maps to 400.
let parsed = CalendarEvent::parse_all_events(calendar_id, &dto.ical_data)?;
let mut out = Vec::with_capacity(parsed.len());
let mut any_inserted = false;
for event in parsed {
let ical_uid = event.ical_uid().to_string();
// Existing row lookup routes on the master/exception split.
// Master: (calendar_id, ical_uid) WHERE recurrence_id IS NULL
// Exception: (calendar_id, ical_uid, recurrence_id)
let existing = match event.recurrence_id().copied() {
Some(rid) => {
self.event_repository
.find_event_by_ical_uid_and_recurrence_id(&calendar_id, &ical_uid, &rid)
.await?
}
None => {
self.event_repository
.find_event_by_ical_uid(&calendar_id, &ical_uid)
.await?
}
};
// Delete-then-insert keeps the DB-level partial unique
// indexes happy and matches the pre-#528 update semantics
// of the single-event path (fresh row id per replace,
// ETag changes on update).
if let Some(existing_event) = existing {
self.event_repository
.delete_event(existing_event.id())
.await?;
} else {
any_inserted = true;
}
let created = self.event_repository.create_event(event).await?;
out.push(CalendarEventDto::from(created));
}
Ok(UpsertEventsResult {
events: out,
any_inserted,
})
}
async fn update_event(
&self,
event_id: &str,
@@ -402,6 +399,18 @@ impl CalendarStoragePort for CalendarStorageAdapter {
Ok(CalendarEventDto::from(event))
}
async fn calendar_id_for_event(&self, event_id: &str) -> Result<String, DomainError> {
let uuid = Uuid::parse_str(event_id).map_err(|_| {
DomainError::new(ErrorKind::InvalidInput, "Event", "Invalid event ID format")
})?;
let calendar_id = self
.event_repository
.find_calendar_id_by_event_id(&uuid)
.await?;
Ok(calendar_id.to_string())
}
async fn find_event_by_ical_uid(
&self,
calendar_id: &str,
@@ -458,6 +467,30 @@ impl CalendarStoragePort for CalendarStorageAdapter {
Ok(events.into_iter().map(CalendarEventDto::from).collect())
}
fn stream_events_uid_order(
&self,
calendar_id: &str,
) -> futures::stream::BoxStream<'static, Result<CalendarEventDto, DomainError>> {
use futures::StreamExt;
let uuid = match Uuid::parse_str(calendar_id) {
Ok(u) => u,
Err(_) => {
return Box::pin(futures::stream::once(async {
Err(DomainError::new(
ErrorKind::InvalidInput,
"Calendar",
"Invalid calendar ID format",
))
}));
}
};
Box::pin(
self.event_repository
.stream_events_uid_order(uuid)
.map(|r| r.map(CalendarEventDto::from)),
)
}
async fn list_events_by_calendar_paginated(
&self,
calendar_id: &str,
File diff suppressed because it is too large Load Diff
@@ -95,6 +95,11 @@ impl MusicStoragePort for MusicStorageAdapter {
}
}
async fn get_playlists_by_ids(&self, ids: &[Uuid]) -> Result<Vec<PlaylistDto>, DomainError> {
let playlists = self.playlist_repository.find_playlists_by_ids(ids).await?;
Ok(playlists.into_iter().map(PlaylistDto::from).collect())
}
async fn list_playlists_by_owner(
&self,
owner_id: Uuid,
@@ -135,19 +140,18 @@ impl MusicStoragePort for MusicStorageAdapter {
limit: i64,
offset: i64,
) -> Result<Vec<PlaylistDto>, DomainError> {
// One `LEFT JOIN … GROUP BY` instead of 1 listing + N per-playlist
// `COUNT(*)` round-trips (up to 101 at limit=100) — benches/ROUND25.md §Q1.
let playlists = self
.playlist_repository
.list_public_playlists(limit, offset)
.list_public_playlists_with_counts(limit, offset)
.await?;
let mut result = Vec::new();
for playlist in playlists {
let dto = PlaylistDto::from(playlist);
let track_count = self
.get_track_count(&uuid::Uuid::parse_str(&dto.id).unwrap())
.await?;
result.push(dto.with_track_info(track_count, 0));
}
Ok(result)
Ok(playlists
.into_iter()
.map(|(playlist, track_count)| {
PlaylistDto::from(playlist).with_track_info(track_count, 0)
})
.collect())
}
async fn user_has_access(&self, playlist_id: &str, user_id: Uuid) -> Result<bool, DomainError> {
+8
View File
@@ -52,6 +52,14 @@ pub async fn create_auth_services(
// direct FolderService dependency for that path.
auth_app_service = auth_app_service.with_user_lifecycle(user_lifecycle);
// Wire the auth-method allowlist + email-verification requirement so
// login / magic-link / register handlers consult a single snapshot
// rather than reaching into the app config on every call.
auth_app_service = auth_app_service.with_auth_policy(
config.auth.allowed_auth_methods.clone(),
config.auth.require_verified_email,
);
// Wire the magic-link token repo. Enables `GET /magic/v1/{token}`
// and the future `POST /api/auth/magic-link/send` endpoint to mint
// and consume tokens. The repo is unconditional (it's just SQL on
@@ -16,6 +16,23 @@ impl AddressBookPgRepository {
pub fn new(pool: Arc<PgPool>) -> Self {
Self { pool }
}
/// `EXISTS` short-circuit for the login provisioning hook — the old
/// `get_address_books_by_owner(..).is_empty()` hydrated every owned
/// `AddressBook` row on EVERY login just to test emptiness (the ROUND9
/// §7 COUNT→EXISTS pattern; benches/ROUND13.md §Q2).
pub async fn has_owned_address_book(&self, owner_id: Uuid) -> Result<bool, DomainError> {
let exists: bool = sqlx::query_scalar(
"SELECT EXISTS(SELECT 1 FROM carddav.address_books WHERE owner_id = $1)",
)
.bind(owner_id)
.fetch_one(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to probe owned address books: {}", e))
})?;
Ok(exists)
}
}
impl AddressBookRepository for AddressBookPgRepository {
@@ -110,6 +127,45 @@ impl AddressBookRepository for AddressBookPgRepository {
Ok(())
}
async fn get_address_books_by_ids(
&self,
ids: &[Uuid],
) -> AddressBookRepositoryResult<Vec<AddressBook>> {
if ids.is_empty() {
return Ok(Vec::new());
}
let rows = sqlx::query(
r#"
SELECT id, name, owner_id, description, color, is_public, created_at, updated_at
FROM carddav.address_books
WHERE id = ANY($1)
"#,
)
.bind(ids)
.fetch_all(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to get address books by ids: {}", e))
})?;
Ok(rows
.iter()
.map(|row| {
let owner_id: Uuid = row.get("owner_id");
AddressBook::from_raw(
row.get("id"),
row.get("name"),
owner_id.to_string(),
row.get("description"),
row.get("color"),
row.get("is_public"),
row.get("created_at"),
row.get("updated_at"),
)
})
.collect())
}
async fn get_address_book_by_id(
&self,
id: &Uuid,
@@ -184,44 +240,6 @@ impl AddressBookRepository for AddressBookPgRepository {
Ok(result)
}
async fn get_shared_address_books(
&self,
user_id: Uuid,
) -> AddressBookRepositoryResult<Vec<AddressBook>> {
let rows = sqlx::query(
r#"
SELECT a.id, a.name, a.owner_id, a.description, a.color, a.is_public, a.created_at, a.updated_at
FROM carddav.address_books a
INNER JOIN carddav.address_book_shares s ON a.id = s.address_book_id
WHERE s.user_id = $1
ORDER BY a.name
"#
)
.bind(user_id)
.fetch_all(&*self.pool)
.await
.map_err(|e| DomainError::database_error(format!("Failed to get shared address books: {}", e)))?;
let result = rows
.into_iter()
.map(|row| {
let owner_id: Uuid = row.get("owner_id");
AddressBook::from_raw(
row.get("id"),
row.get("name"),
owner_id.to_string(),
row.get("description"),
row.get("color"),
row.get("is_public"),
row.get("created_at"),
row.get("updated_at"),
)
})
.collect();
Ok(result)
}
async fn get_public_address_books(&self) -> AddressBookRepositoryResult<Vec<AddressBook>> {
let rows = sqlx::query(
r#"
@@ -256,79 +274,4 @@ impl AddressBookRepository for AddressBookPgRepository {
Ok(result)
}
async fn share_address_book(
&self,
address_book_id: &Uuid,
user_id: Uuid,
can_write: bool,
) -> AddressBookRepositoryResult<()> {
sqlx::query(
r#"
INSERT INTO carddav.address_book_shares (address_book_id, user_id, can_write)
VALUES ($1, $2, $3)
ON CONFLICT (address_book_id, user_id) DO UPDATE SET can_write = $3
"#,
)
.bind(address_book_id)
.bind(user_id)
.bind(can_write)
.execute(&*self.pool)
.await
.map_err(|e| DomainError::database_error(format!("Failed to share address book: {}", e)))?;
Ok(())
}
async fn unshare_address_book(
&self,
address_book_id: &Uuid,
user_id: Uuid,
) -> AddressBookRepositoryResult<()> {
sqlx::query(
r#"
DELETE FROM carddav.address_book_shares
WHERE address_book_id = $1 AND user_id = $2
"#,
)
.bind(address_book_id)
.bind(user_id)
.execute(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to unshare address book: {}", e))
})?;
Ok(())
}
async fn get_address_book_shares(
&self,
address_book_id: &Uuid,
) -> AddressBookRepositoryResult<Vec<(String, bool)>> {
let rows = sqlx::query(
r#"
SELECT user_id, can_write
FROM carddav.address_book_shares
WHERE address_book_id = $1
ORDER BY user_id
"#,
)
.bind(address_book_id)
.fetch_all(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to get address book shares: {}", e))
})?;
let result = rows
.into_iter()
.map(|row| {
let user_id: Uuid = row.get("user_id");
(user_id.to_string(), row.get("can_write"))
})
.collect();
Ok(result)
}
}
@@ -16,6 +16,31 @@ impl CalendarEventPgRepository {
pub fn new(pool: Arc<PgPool>) -> Self {
Self { pool }
}
/// Shared row → entity mapping (the inline shape every listing
/// method uses, factored for the cursor stream).
fn row_to_event(row: &sqlx::postgres::PgRow) -> CalendarEventRepositoryResult<CalendarEvent> {
let mut event = CalendarEvent::with_id(
row.get("id"),
row.get("calendar_id"),
row.get("summary"),
row.get::<Option<String>, _>("description"),
row.get::<Option<String>, _>("location"),
row.get("start_time"),
row.get("end_time"),
row.get("all_day"),
row.get::<Option<String>, _>("rrule"),
row.get("ical_uid"),
row.get("ical_data"),
row.get("created_at"),
row.get("updated_at"),
)
.map_err(|e| {
DomainError::database_error(format!("Error creating calendar event: {}", e))
})?;
event.set_recurrence_id(row.get::<Option<DateTime<Utc>>, _>("recurrence_id"));
Ok(event)
}
}
impl CalendarEventRepository for CalendarEventPgRepository {
@@ -30,10 +55,11 @@ impl CalendarEventRepository for CalendarEventPgRepository {
sqlx::query(
r#"
INSERT INTO caldav.calendar_events (
id, calendar_id, summary, description, location, start_time, end_time,
all_day, rrule, created_at, updated_at, ical_uid, ical_data
id, calendar_id, summary, description, location, start_time, end_time,
all_day, rrule, created_at, updated_at, ical_uid, ical_data,
recurrence_id
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
"#,
)
.bind(event.id())
@@ -49,6 +75,10 @@ impl CalendarEventRepository for CalendarEventPgRepository {
.bind(event.updated_at())
.bind(event.ical_uid())
.bind(event.ical_data())
// NULL on masters, non-NULL on exception overrides — see the
// `20260913000001_calendar_events_recurrence_id.sql` migration
// and `docs/architecture/rebac-authorization.md` follow-up doc.
.bind(event.recurrence_id().copied())
.execute(&*self.pool)
.await
.map_err(|e| {
@@ -68,16 +98,17 @@ impl CalendarEventRepository for CalendarEventPgRepository {
sqlx::query(
r#"
UPDATE caldav.calendar_events
SET summary = $1,
description = $2,
location = $3,
start_time = $4,
end_time = $5,
all_day = $6,
SET summary = $1,
description = $2,
location = $3,
start_time = $4,
end_time = $5,
all_day = $6,
rrule = $7,
ical_data = $8,
updated_at = $9
WHERE id = $10
recurrence_id = $9,
updated_at = $10
WHERE id = $11
"#,
)
.bind(event.summary())
@@ -88,6 +119,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
.bind(event.all_day())
.bind(event.rrule())
.bind(event.ical_data())
.bind(event.recurrence_id().copied())
.bind(now)
.bind(event.id())
.execute(&*self.pool)
@@ -126,12 +158,12 @@ impl CalendarEventRepository for CalendarEventPgRepository {
) -> CalendarEventRepositoryResult<Vec<CalendarEvent>> {
let rows = sqlx::query(
r#"
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data, recurrence_id
FROM caldav.calendar_events
WHERE calendar_id = $1
WHERE calendar_id = $1
AND (
(start_time >= $2 AND start_time < $3) OR
(end_time > $2 AND end_time <= $3) OR
@@ -150,9 +182,9 @@ impl CalendarEventRepository for CalendarEventPgRepository {
DomainError::database_error(format!("Failed to get events in time range: {}", e))
})?;
let mut events = Vec::new();
let mut events = Vec::with_capacity(rows.len());
for row in rows {
let event = CalendarEvent::with_id(
let mut event = CalendarEvent::with_id(
row.get("id"),
row.get("calendar_id"),
row.get("summary"),
@@ -170,19 +202,35 @@ impl CalendarEventRepository for CalendarEventPgRepository {
.map_err(|e| {
DomainError::database_error(format!("Error creating calendar event: {}", e))
})?;
// Rehydrate the RECURRENCE-ID after entity construction —
// `with_id` initialises to `None` because the field predates
// the rest of the constructor signature (#528). Keeping
// `with_id` unchanged avoids ripple-changing every caller.
event.set_recurrence_id(row.get::<Option<DateTime<Utc>>, _>("recurrence_id"));
events.push(event);
}
Ok(events)
}
async fn find_calendar_id_by_event_id(&self, id: &Uuid) -> CalendarEventRepositoryResult<Uuid> {
sqlx::query_scalar("SELECT calendar_id FROM caldav.calendar_events WHERE id = $1")
.bind(id)
.fetch_optional(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to get event calendar id: {}", e))
})?
.ok_or_else(|| DomainError::not_found("Calendar Event", id.to_string()))
}
async fn find_event_by_id(&self, id: &Uuid) -> CalendarEventRepositoryResult<CalendarEvent> {
let row = sqlx::query(
r#"
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data, recurrence_id
FROM caldav.calendar_events
WHERE id = $1
"#,
@@ -195,11 +243,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
})?
.ok_or_else(|| DomainError::not_found("Calendar Event", id.to_string()))?;
// In a real implementation, we would build a complete CalendarEvent object
// For simplicity, we create an object with default values to
// demonstrate the approach without macros
let event = CalendarEvent::with_id(
let mut event = CalendarEvent::with_id(
row.get("id"),
row.get("calendar_id"),
row.get("summary"),
@@ -217,6 +261,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
.map_err(|e| {
DomainError::database_error(format!("Error creating calendar event: {}", e))
})?;
event.set_recurrence_id(row.get::<Option<DateTime<Utc>>, _>("recurrence_id"));
Ok(event)
}
@@ -227,10 +272,10 @@ impl CalendarEventRepository for CalendarEventPgRepository {
) -> CalendarEventRepositoryResult<Vec<CalendarEvent>> {
let rows = sqlx::query(
r#"
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data, recurrence_id
FROM caldav.calendar_events
WHERE calendar_id = $1
ORDER BY start_time
@@ -243,9 +288,9 @@ impl CalendarEventRepository for CalendarEventPgRepository {
DomainError::database_error(format!("Failed to get events by calendar: {}", e))
})?;
let mut events = Vec::new();
let mut events = Vec::with_capacity(rows.len());
for row in rows {
let event = CalendarEvent::with_id(
let mut event = CalendarEvent::with_id(
row.get("id"),
row.get("calendar_id"),
row.get("summary"),
@@ -263,6 +308,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
.map_err(|e| {
DomainError::database_error(format!("Error creating calendar event: {}", e))
})?;
event.set_recurrence_id(row.get::<Option<DateTime<Utc>>, _>("recurrence_id"));
events.push(event);
}
@@ -278,10 +324,10 @@ impl CalendarEventRepository for CalendarEventPgRepository {
let rows = sqlx::query(
r#"
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data, recurrence_id
FROM caldav.calendar_events
WHERE calendar_id = $1 AND summary ILIKE $2
ORDER BY start_time
@@ -295,9 +341,9 @@ impl CalendarEventRepository for CalendarEventPgRepository {
DomainError::database_error(format!("Failed to find events by summary: {}", e))
})?;
let mut events = Vec::new();
let mut events = Vec::with_capacity(rows.len());
for row in rows {
let event = CalendarEvent::with_id(
let mut event = CalendarEvent::with_id(
row.get("id"),
row.get("calendar_id"),
row.get("summary"),
@@ -315,6 +361,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
.map_err(|e| {
DomainError::database_error(format!("Error creating calendar event: {}", e))
})?;
event.set_recurrence_id(row.get::<Option<DateTime<Utc>>, _>("recurrence_id"));
events.push(event);
}
@@ -326,14 +373,21 @@ impl CalendarEventRepository for CalendarEventPgRepository {
calendar_id: &Uuid,
ical_uid: &str,
) -> CalendarEventRepositoryResult<Option<CalendarEvent>> {
// Phase 2 note: this method looks up "an event with this UID"
// — the SELECT still isn't filtered on `recurrence_id IS NULL`
// because the phase-3 handler routing (which will distinguish
// master vs. exception override at PUT time) is where the
// filter actually needs to live. For phase 2 the invariant is
// enforced only at INSERT time via the two partial unique
// indexes; reads see whatever's there.
let row_opt = sqlx::query(
r#"
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data, recurrence_id
FROM caldav.calendar_events
WHERE calendar_id = $1 AND ical_uid = $2
WHERE calendar_id = $1 AND ical_uid = $2 AND recurrence_id IS NULL
"#,
)
.bind(calendar_id)
@@ -346,7 +400,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
match row_opt {
Some(row) => {
let event = CalendarEvent::with_id(
let mut event = CalendarEvent::with_id(
row.get("id"),
row.get("calendar_id"),
row.get("summary"),
@@ -364,6 +418,67 @@ impl CalendarEventRepository for CalendarEventPgRepository {
.map_err(|e| {
DomainError::database_error(format!("Error creating calendar event: {}", e))
})?;
event.set_recurrence_id(row.get::<Option<DateTime<Utc>>, _>("recurrence_id"));
Ok(Some(event))
}
None => Ok(None),
}
}
async fn find_event_by_ical_uid_and_recurrence_id(
&self,
calendar_id: &Uuid,
ical_uid: &str,
recurrence_id: &DateTime<Utc>,
) -> CalendarEventRepositoryResult<Option<CalendarEvent>> {
// Uses idx_calendar_events_exception_unique — the partial
// unique index on (calendar_id, ical_uid, recurrence_id)
// WHERE recurrence_id IS NOT NULL — for the exact-match seek.
let row_opt = sqlx::query(
r#"
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data, recurrence_id
FROM caldav.calendar_events
WHERE calendar_id = $1
AND ical_uid = $2
AND recurrence_id = $3
"#,
)
.bind(calendar_id)
.bind(ical_uid)
.bind(recurrence_id)
.fetch_optional(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!(
"Failed to get calendar event exception by UID+RECURRENCE-ID: {}",
e
))
})?;
match row_opt {
Some(row) => {
let mut event = CalendarEvent::with_id(
row.get("id"),
row.get("calendar_id"),
row.get("summary"),
row.get::<Option<String>, _>("description"),
row.get::<Option<String>, _>("location"),
row.get("start_time"),
row.get("end_time"),
row.get("all_day"),
row.get::<Option<String>, _>("rrule"),
row.get("ical_uid"),
row.get("ical_data"),
row.get("created_at"),
row.get("updated_at"),
)
.map_err(|e| {
DomainError::database_error(format!("Error creating calendar event: {}", e))
})?;
event.set_recurrence_id(row.get::<Option<DateTime<Utc>>, _>("recurrence_id"));
Ok(Some(event))
}
None => Ok(None),
@@ -375,12 +490,18 @@ impl CalendarEventRepository for CalendarEventPgRepository {
calendar_id: &Uuid,
ical_uids: &[String],
) -> CalendarEventRepositoryResult<Vec<CalendarEvent>> {
// Batch UID lookup returns ALL rows for the given UIDs, both
// masters and exception overrides. Callers that want just
// masters filter downstream. Same phase-2 policy as the
// single-UID variant — read-side filtering is a phase-3
// concern; the DB unique indexes are what guarantee at most
// one master + N distinct exceptions per (calendar, UID).
let rows = sqlx::query(
r#"
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data
created_at, updated_at, ical_uid, ical_data, recurrence_id
FROM caldav.calendar_events
WHERE calendar_id = $1 AND ical_uid = ANY($2)
ORDER BY start_time
@@ -394,9 +515,9 @@ impl CalendarEventRepository for CalendarEventPgRepository {
DomainError::database_error(format!("Failed to get calendar events by UIDs: {}", e))
})?;
let mut events = Vec::new();
let mut events = Vec::with_capacity(rows.len());
for row in rows {
let event = CalendarEvent::with_id(
let mut event = CalendarEvent::with_id(
row.get("id"),
row.get("calendar_id"),
row.get("summary"),
@@ -414,6 +535,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
.map_err(|e| {
DomainError::database_error(format!("Error creating calendar event: {}", e))
})?;
event.set_recurrence_id(row.get::<Option<DateTime<Utc>>, _>("recurrence_id"));
events.push(event);
}
@@ -461,6 +583,57 @@ impl CalendarEventRepository for CalendarEventPgRepository {
Ok(result.rows_affected() as i64)
}
fn stream_events_uid_order(
&self,
calendar_id: Uuid,
) -> futures::stream::BoxStream<'static, CalendarEventRepositoryResult<CalendarEvent>> {
// ONE ordered scan for the whole calendar, served through a PG
// cursor (`fetch`) so only a window of rows is in flight. The
// window function puts every UID's rows adjacent, bundles
// ordered by first occurrence — exactly the first-appearance
// order the buffered `ORDER BY start_time` listing produced
// after grouping — with the master row first inside each UID.
//
// The first streaming shape hydrated pages via
// `ical_uid = ANY(page)`: ~20 µs per index descent made the
// total wall 3-4x the buffered single scan (measured in
// benches/ROUND5.md). This keeps the buffered path's one
// scan+sort while bounding memory to a page.
let pool = self.pool.clone();
let stream: futures::stream::BoxStream<
'static,
CalendarEventRepositoryResult<CalendarEvent>,
> = Box::pin(async_stream::try_stream! {
let mut conn = pool.acquire().await.map_err(|e| {
DomainError::database_error(format!("Failed to acquire connection: {}", e))
})?;
let mut rows = sqlx::query(
r#"
SELECT
id, calendar_id, summary, description, location,
start_time, end_time, all_day, rrule,
created_at, updated_at, ical_uid, ical_data, recurrence_id
FROM caldav.calendar_events
WHERE calendar_id = $1
ORDER BY MIN(start_time) OVER (PARTITION BY ical_uid),
ical_uid,
(recurrence_id IS NOT NULL),
start_time
"#,
)
.bind(calendar_id)
.fetch(&mut *conn);
use futures::TryStreamExt;
while let Some(row) = rows.try_next().await.map_err(|e| {
DomainError::database_error(format!("Failed to stream events: {}", e))
})? {
yield Self::row_to_event(&row)?;
}
});
stream
}
async fn list_events_by_calendar_paginated(
&self,
calendar_id: &Uuid,
@@ -491,7 +664,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
))
})?;
let mut events = Vec::new();
let mut events = Vec::with_capacity(rows.len());
for row in rows {
let event = CalendarEvent::with_id(
row.get("id"),
@@ -546,7 +719,7 @@ impl CalendarEventRepository for CalendarEventPgRepository {
DomainError::database_error(format!("Failed to find recurring events in range: {}", e))
})?;
let mut events = Vec::new();
let mut events = Vec::with_capacity(rows.len());
for row in rows {
let event = CalendarEvent::with_id(
row.get("id"),
@@ -16,6 +16,24 @@ impl CalendarPgRepository {
pub fn new(pool: Arc<PgPool>) -> Self {
Self { pool }
}
/// `EXISTS` short-circuit for the login provisioning hook, which only
/// needs to know whether the user owns ANY calendar. The old
/// `list_calendars_by_owner(..).is_empty()` hydrated every owned
/// `Calendar` row (8 cols incl. description/color TEXT) on EVERY login
/// just to test emptiness — the ROUND9 §7 `Drive::is_empty` COUNT→EXISTS
/// pattern (benches/ROUND13.md §Q2).
pub async fn has_owned_calendar(&self, owner_id: Uuid) -> CalendarRepositoryResult<bool> {
let exists: bool =
sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM caldav.calendars WHERE owner_id = $1)")
.bind(owner_id)
.fetch_one(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to probe owned calendars: {}", e))
})?;
Ok(exists)
}
}
impl CalendarRepository for CalendarPgRepository {
@@ -138,6 +156,42 @@ impl CalendarRepository for CalendarPgRepository {
Ok(calendar)
}
async fn find_calendars_by_ids(&self, ids: &[Uuid]) -> CalendarRepositoryResult<Vec<Calendar>> {
if ids.is_empty() {
return Ok(Vec::new());
}
let rows = sqlx::query(
r#"
SELECT id, name, owner_id, description, color, is_public, created_at, updated_at
FROM caldav.calendars
WHERE id = ANY($1)
"#,
)
.bind(ids)
.fetch_all(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to get calendars by ids: {}", e))
})?;
rows.iter()
.map(|row| {
Calendar::with_id(
row.get("id"),
row.get("name"),
row.get("owner_id"),
row.get("description"),
row.get("color"),
row.get("created_at"),
row.get("updated_at"),
)
.map_err(|e| {
DomainError::database_error(format!("Failed to create calendar object: {}", e))
})
})
.collect()
}
async fn list_calendars_by_owner(
&self,
owner_id: Uuid,
@@ -157,7 +211,7 @@ impl CalendarRepository for CalendarPgRepository {
DomainError::database_error(format!("Failed to get calendars by owner: {}", e))
})?;
let mut calendars = Vec::new();
let mut calendars = Vec::with_capacity(rows.len());
for row in rows {
let calendar = Calendar::with_id(
row.get("id"),
@@ -216,44 +270,6 @@ impl CalendarRepository for CalendarPgRepository {
Ok(calendar)
}
async fn list_calendars_shared_with_user(
&self,
user_id: Uuid,
) -> CalendarRepositoryResult<Vec<Calendar>> {
let rows = sqlx::query(
r#"
SELECT c.id, c.name, c.owner_id, c.description, c.color, c.is_public, c.created_at, c.updated_at
FROM caldav.calendars c
INNER JOIN caldav.calendar_shares s ON c.id = s.calendar_id
WHERE s.user_id = $1
ORDER BY c.name
"#
)
.bind(user_id)
.fetch_all(&*self.pool)
.await
.map_err(|e| DomainError::database_error(format!("Failed to get shared calendars: {}", e)))?;
let mut calendars = Vec::new();
for row in rows {
let calendar = Calendar::with_id(
row.get("id"),
row.get("name"),
row.get("owner_id"),
row.get("description"),
row.get("color"),
row.get("created_at"),
row.get("updated_at"),
)
.map_err(|e| {
DomainError::database_error(format!("Failed to create calendar object: {}", e))
})?;
calendars.push(calendar);
}
Ok(calendars)
}
async fn list_public_calendars(
&self,
limit: i64,
@@ -276,7 +292,7 @@ impl CalendarRepository for CalendarPgRepository {
DomainError::database_error(format!("Failed to get public calendars: {}", e))
})?;
let mut calendars = Vec::new();
let mut calendars = Vec::with_capacity(rows.len());
for row in rows {
let calendar = Calendar::with_id(
row.get("id"),
@@ -296,112 +312,6 @@ impl CalendarRepository for CalendarPgRepository {
Ok(calendars)
}
async fn user_has_calendar_access(
&self,
calendar_id: &Uuid,
user_id: Uuid,
) -> CalendarRepositoryResult<bool> {
// Check if the user is the owner of the calendar or has a share
let row = sqlx::query(
r#"
SELECT EXISTS (
SELECT 1 FROM caldav.calendars c
WHERE c.id = $1 AND (c.owner_id = $2 OR c.is_public = true)
UNION
SELECT 1 FROM caldav.calendar_shares s
WHERE s.calendar_id = $1 AND s.user_id = $2
) as has_access
"#,
)
.bind(calendar_id)
.bind(user_id)
.fetch_one(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to check calendar access: {}", e))
})?;
Ok(row.get::<bool, _>("has_access"))
}
async fn share_calendar(
&self,
calendar_id: &Uuid,
user_id: Uuid,
access_level: &str,
) -> CalendarRepositoryResult<()> {
// Validate access level
if !["read", "write", "owner"].contains(&access_level) {
return Err(DomainError::validation_error(format!(
"Invalid access level: '{}'. Must be 'read', 'write', or 'owner'",
access_level
)));
}
sqlx::query(
r#"
INSERT INTO caldav.calendar_shares (calendar_id, user_id, access_level)
VALUES ($1, $2, $3)
ON CONFLICT (calendar_id, user_id) DO UPDATE SET access_level = $3
"#,
)
.bind(calendar_id)
.bind(user_id)
.bind(access_level)
.execute(&*self.pool)
.await
.map_err(|e| DomainError::database_error(format!("Failed to share calendar: {}", e)))?;
Ok(())
}
async fn remove_calendar_sharing(
&self,
calendar_id: &Uuid,
user_id: Uuid,
) -> CalendarRepositoryResult<()> {
sqlx::query(
r#"
DELETE FROM caldav.calendar_shares
WHERE calendar_id = $1 AND user_id = $2
"#,
)
.bind(calendar_id)
.bind(user_id)
.execute(&*self.pool)
.await
.map_err(|e| DomainError::database_error(format!("Failed to unshare calendar: {}", e)))?;
Ok(())
}
async fn get_calendar_shares(
&self,
calendar_id: &Uuid,
) -> CalendarRepositoryResult<Vec<(String, String)>> {
let rows = sqlx::query(
r#"
SELECT user_id, access_level
FROM caldav.calendar_shares
WHERE calendar_id = $1
ORDER BY user_id
"#,
)
.bind(calendar_id)
.fetch_all(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to get calendar shares: {}", e))
})?;
let mut shares = Vec::new();
for row in rows {
shares.push((row.get("user_id"), row.get("access_level")));
}
Ok(shares)
}
async fn get_calendar_property(
&self,
calendar_id: &Uuid,
@@ -490,7 +400,7 @@ impl CalendarRepository for CalendarPgRepository {
DomainError::database_error(format!("Failed to get calendar properties: {}", e))
})?;
let mut properties = std::collections::HashMap::new();
let mut properties = std::collections::HashMap::with_capacity(rows.len());
for row in rows {
properties.insert(row.get("name"), row.get("value"));
}
@@ -1,5 +1,4 @@
use chrono::Utc;
use serde_json::Value as JsonValue;
use sqlx::{PgPool, Row, types::Uuid};
use std::sync::Arc;
@@ -177,16 +176,30 @@ impl ContactGroupRepository for ContactGroupPgRepository {
Ok(())
}
async fn count_contacts_in_group(&self, group_id: &Uuid) -> ContactRepositoryResult<i64> {
sqlx::query_scalar("SELECT COUNT(*) FROM carddav.group_memberships WHERE group_id = $1")
.bind(group_id)
.fetch_one(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::new(
ErrorKind::InternalError,
"ContactGroup",
format!("Failed to count contacts in group: {}", e),
)
})
}
async fn get_contacts_in_group(
&self,
group_id: &Uuid,
) -> ContactRepositoryResult<Vec<Contact>> {
let rows = sqlx::query(
r#"
SELECT
SELECT
c.id, c.address_book_id, c.uid, c.full_name, c.first_name, c.last_name, c.nickname,
c.email, c.phone, c.address, c.organization, c.title, c.notes, c.photo_url,
c.birthday, c.anniversary, c.vcard, c.etag, c.created_at, c.updated_at
c.birthday, c.anniversary, c.etag, c.created_at, c.updated_at
FROM carddav.contacts c
INNER JOIN carddav.group_memberships gm ON c.id = gm.contact_id
WHERE gm.group_id = $1
@@ -204,20 +217,23 @@ impl ContactGroupRepository for ContactGroupPgRepository {
)
})?;
let mut contacts = Vec::new();
let mut contacts = Vec::with_capacity(rows.len());
for row in &rows {
let email_json: JsonValue = row.get("email");
let phone_json: JsonValue = row.get("phone");
let address_json: JsonValue = row.get("address");
let emails = serde_json::from_value::<Vec<EmailPersistenceDto>>(email_json)
.map(emails_from_persistence)
// Typed `Json<T>` decode (one `from_slice` pass) instead of the
// `Value` DOM + `from_value` re-walk — the contact_pg_repository
// §J1 fix applied to this inlined sibling. Byte-identical result,
// 3 fewer throwaway DOMs per contact. (benches/ROUND23.md §J1)
let emails = row
.try_get::<sqlx::types::Json<Vec<EmailPersistenceDto>>, _>("email")
.map(|j| emails_from_persistence(j.0))
.unwrap_or_default();
let phones = serde_json::from_value::<Vec<PhonePersistenceDto>>(phone_json)
.map(phones_from_persistence)
let phones = row
.try_get::<sqlx::types::Json<Vec<PhonePersistenceDto>>, _>("phone")
.map(|j| phones_from_persistence(j.0))
.unwrap_or_default();
let addresses = serde_json::from_value::<Vec<AddressPersistenceDto>>(address_json)
.map(addresses_from_persistence)
let addresses = row
.try_get::<sqlx::types::Json<Vec<AddressPersistenceDto>>, _>("address")
.map(|j| addresses_from_persistence(j.0))
.unwrap_or_default();
contacts.push(Contact::from_raw(
@@ -237,7 +253,13 @@ impl ContactGroupRepository for ContactGroupPgRepository {
row.get::<Option<String>, _>("photo_url"),
row.get("birthday"),
row.get("anniversary"),
row.get("vcard"),
// vcard column intentionally NOT selected — the sole live caller
// (`list_contacts_in_group`) maps to `ContactDto`, which has no
// vcard field, so fetching the multi-KB serialized vCard (with an
// embedded base64 PHOTO) only to drop it wastes bandwidth + a
// per-row String. Mirrors `row_to_contact_lite` (benches/ROUND29.md
// §F / ROUND25 §Q2, applied to the LIVE group method this time).
String::new(),
row.get("etag"),
row.get("created_at"),
row.get("updated_at"),
@@ -1,5 +1,4 @@
use chrono::Utc;
use serde_json::Value as JsonValue;
use sqlx::{PgPool, Row, types::Uuid};
use std::sync::Arc;
@@ -21,20 +20,49 @@ impl ContactPgRepository {
Self { pool }
}
/// Maps a database row to a Contact domain entity
/// Maps a database row to a Contact domain entity (reads the `vcard` column).
fn row_to_contact(row: &sqlx::postgres::PgRow) -> Result<Contact, DomainError> {
let email_json: JsonValue = row.get("email");
let phone_json: JsonValue = row.get("phone");
let address_json: JsonValue = row.get("address");
Self::row_to_contact_with_vcard(row, row.get("vcard"))
}
let emails = serde_json::from_value::<Vec<EmailPersistenceDto>>(email_json)
.map(emails_from_persistence)
/// Maps a row whose SELECT omitted the `vcard` column — used by the REST
/// listings (paginated / search / by-group) whose `ContactDto` drops vcard
/// anyway, so the multi-KB vCard TEXT (which can embed a base64 PHOTO) is
/// never SELECTed, shipped over the wire, or allocated (benches/ROUND25.md
/// §Q2). The domain `Contact` keeps an empty vcard; these paths never
/// re-emit it. Do NOT use for CardDAV sync / whole-book export, which need
/// the round-trip vCard.
fn row_to_contact_lite(row: &sqlx::postgres::PgRow) -> Result<Contact, DomainError> {
Self::row_to_contact_with_vcard(row, String::new())
}
/// Shared row → `Contact` mapper; `vcard` is supplied by the caller so the
/// TEXT column can be omitted from listings that don't consume it.
fn row_to_contact_with_vcard(
row: &sqlx::postgres::PgRow,
vcard: String,
) -> Result<Contact, DomainError> {
// Decode each JSONB column straight into its typed Vec via
// `sqlx::types::Json<T>` (a single `serde_json::from_slice` pass over
// the raw JSONB bytes) instead of `row.get::<serde_json::Value>` +
// `serde_json::from_value`, which built a throwaway `Value` DOM per
// column and then walked it a SECOND time to produce the typed Vec —
// 3 discarded DOMs per contact row on every list / multiget / CardDAV
// sync. `try_get` preserves the exact malformed-shape fallback (the old
// `from_value(...).unwrap_or_default()`; a bare `row.get` would panic on
// a decode error); the columns are `JSONB NOT NULL DEFAULT '[]'`, so SQL
// NULL never occurs. (benches/ROUND23.md §J1)
let emails = row
.try_get::<sqlx::types::Json<Vec<EmailPersistenceDto>>, _>("email")
.map(|j| emails_from_persistence(j.0))
.unwrap_or_default();
let phones = serde_json::from_value::<Vec<PhonePersistenceDto>>(phone_json)
.map(phones_from_persistence)
let phones = row
.try_get::<sqlx::types::Json<Vec<PhonePersistenceDto>>, _>("phone")
.map(|j| phones_from_persistence(j.0))
.unwrap_or_default();
let addresses = serde_json::from_value::<Vec<AddressPersistenceDto>>(address_json)
.map(addresses_from_persistence)
let addresses = row
.try_get::<sqlx::types::Json<Vec<AddressPersistenceDto>>, _>("address")
.map(|j| addresses_from_persistence(j.0))
.unwrap_or_default();
Ok(Contact::from_raw(
@@ -54,7 +82,7 @@ impl ContactPgRepository {
row.get::<Option<String>, _>("photo_url"),
row.get("birthday"),
row.get("anniversary"),
row.get("vcard"),
vcard,
row.get("etag"),
row.get("created_at"),
row.get("updated_at"),
@@ -69,10 +97,6 @@ impl ContactRepository for ContactPgRepository {
let phone_dtos = phones_to_persistence(contact.phone());
let address_dtos = addresses_to_persistence(contact.address());
let email_json = serde_json::to_value(&email_dtos).unwrap_or(JsonValue::Null);
let phone_json = serde_json::to_value(&phone_dtos).unwrap_or(JsonValue::Null);
let address_json = serde_json::to_value(&address_dtos).unwrap_or(JsonValue::Null);
let row = sqlx::query(
r#"
INSERT INTO carddav.contacts (
@@ -97,9 +121,9 @@ impl ContactRepository for ContactPgRepository {
.bind(contact.first_name_owned())
.bind(contact.last_name_owned())
.bind(contact.nickname_owned())
.bind(email_json)
.bind(phone_json)
.bind(address_json)
.bind(sqlx::types::Json(&email_dtos))
.bind(sqlx::types::Json(&phone_dtos))
.bind(sqlx::types::Json(&address_dtos))
.bind(contact.organization_owned())
.bind(contact.title_owned())
.bind(contact.notes_owned())
@@ -124,10 +148,6 @@ impl ContactRepository for ContactPgRepository {
let phone_dtos = phones_to_persistence(contact.phone());
let address_dtos = addresses_to_persistence(contact.address());
let email_json = serde_json::to_value(&email_dtos).unwrap_or(JsonValue::Null);
let phone_json = serde_json::to_value(&phone_dtos).unwrap_or(JsonValue::Null);
let address_json = serde_json::to_value(&address_dtos).unwrap_or(JsonValue::Null);
// Create a clone of the contact with the updated timestamp
let mut updated_contact = contact.clone();
updated_contact.set_updated_at(now);
@@ -163,9 +183,9 @@ impl ContactRepository for ContactPgRepository {
.bind(updated_contact.first_name_owned())
.bind(updated_contact.last_name_owned())
.bind(updated_contact.nickname_owned())
.bind(email_json)
.bind(phone_json)
.bind(address_json)
.bind(sqlx::types::Json(&email_dtos))
.bind(sqlx::types::Json(&phone_dtos))
.bind(sqlx::types::Json(&address_dtos))
.bind(updated_contact.organization_owned())
.bind(updated_contact.title_owned())
.bind(updated_contact.notes_owned())
@@ -271,13 +291,52 @@ impl ContactRepository for ContactPgRepository {
DomainError::database_error(format!("Failed to get contacts by uids: {}", e))
})?;
let mut contacts = Vec::new();
let mut contacts = Vec::with_capacity(rows.len());
for row in &rows {
contacts.push(Self::row_to_contact(row)?);
}
Ok(contacts)
}
fn stream_contacts_by_book(
&self,
address_book_id: Uuid,
) -> futures::stream::BoxStream<'static, ContactRepositoryResult<Contact>> {
// ONE ordered scan served through a PG cursor — the CardDAV
// multistatus emitters page over this stream so only a page of
// contacts is resident (same design as the CalDAV round-5
// cursor; contacts have no master/exception bundling, so pages
// can cut anywhere).
let pool = self.pool.clone();
let stream: futures::stream::BoxStream<'static, ContactRepositoryResult<Contact>> =
Box::pin(async_stream::try_stream! {
let mut conn = pool.acquire().await.map_err(|e| {
DomainError::database_error(format!("Failed to acquire connection: {}", e))
})?;
let mut rows = sqlx::query(
r#"
SELECT
id, address_book_id, uid, full_name, first_name, last_name, nickname,
email, phone, address, organization, title, notes, photo_url,
birthday, anniversary, vcard, etag, created_at, updated_at
FROM carddav.contacts
WHERE address_book_id = $1
ORDER BY full_name, first_name, last_name
"#,
)
.bind(address_book_id)
.fetch(&mut *conn);
use futures::TryStreamExt;
while let Some(row) = rows.try_next().await.map_err(|e| {
DomainError::database_error(format!("Failed to stream contacts: {}", e))
})? {
yield Self::row_to_contact(&row)?;
}
});
stream
}
async fn get_contacts_by_address_book(
&self,
address_book_id: &Uuid,
@@ -300,7 +359,7 @@ impl ContactRepository for ContactPgRepository {
DomainError::database_error(format!("Failed to get contacts by address book: {}", e))
})?;
let mut contacts = Vec::new();
let mut contacts = Vec::with_capacity(rows.len());
for row in &rows {
contacts.push(Self::row_to_contact(row)?);
}
@@ -318,7 +377,7 @@ impl ContactRepository for ContactPgRepository {
SELECT
id, address_book_id, uid, full_name, first_name, last_name, nickname,
email, phone, address, organization, title, notes, photo_url,
birthday, anniversary, vcard, etag, created_at, updated_at
birthday, anniversary, etag, created_at, updated_at
FROM carddav.contacts
WHERE address_book_id = $1
ORDER BY full_name, first_name, last_name
@@ -337,9 +396,9 @@ impl ContactRepository for ContactPgRepository {
))
})?;
let mut contacts = Vec::new();
let mut contacts = Vec::with_capacity(rows.len());
for row in &rows {
contacts.push(Self::row_to_contact(row)?);
contacts.push(Self::row_to_contact_lite(row)?);
}
Ok(contacts)
}
@@ -365,7 +424,7 @@ impl ContactRepository for ContactPgRepository {
DomainError::database_error(format!("Failed to get contacts by email: {}", e))
})?;
let mut contacts = Vec::new();
let mut contacts = Vec::with_capacity(rows.len());
for row in &rows {
contacts.push(Self::row_to_contact(row)?);
}
@@ -381,7 +440,7 @@ impl ContactRepository for ContactPgRepository {
SELECT
c.id, c.address_book_id, c.uid, c.full_name, c.first_name, c.last_name, c.nickname,
c.email, c.phone, c.address, c.organization, c.title, c.notes, c.photo_url,
c.birthday, c.anniversary, c.vcard, c.etag, c.created_at, c.updated_at
c.birthday, c.anniversary, c.etag, c.created_at, c.updated_at
FROM carddav.contacts c
INNER JOIN carddav.group_memberships m ON c.id = m.contact_id
WHERE m.group_id = $1
@@ -395,9 +454,9 @@ impl ContactRepository for ContactPgRepository {
DomainError::database_error(format!("Failed to get contacts by group: {}", e))
})?;
let mut contacts = Vec::new();
let mut contacts = Vec::with_capacity(rows.len());
for row in &rows {
contacts.push(Self::row_to_contact(row)?);
contacts.push(Self::row_to_contact_lite(row)?);
}
Ok(contacts)
}
@@ -414,9 +473,9 @@ impl ContactRepository for ContactPgRepository {
SELECT
id, address_book_id, uid, full_name, first_name, last_name, nickname,
email, phone, address, organization, title, notes, photo_url,
birthday, anniversary, vcard, etag, created_at, updated_at
birthday, anniversary, etag, created_at, updated_at
FROM carddav.contacts
WHERE address_book_id = $1
WHERE address_book_id = $1
AND (
full_name ILIKE $2
OR first_name ILIKE $2
@@ -435,9 +494,9 @@ impl ContactRepository for ContactPgRepository {
.await
.map_err(|e| DomainError::database_error(format!("Failed to search contacts: {}", e)))?;
let mut contacts = Vec::new();
let mut contacts = Vec::with_capacity(rows.len());
for row in &rows {
contacts.push(Self::row_to_contact(row)?);
contacts.push(Self::row_to_contact_lite(row)?);
}
Ok(contacts)
}
@@ -3,14 +3,16 @@
//! The repo deals only with the `storage.drives` table itself. Drive
//! membership lives in `storage.role_grants` (`resource_type='drive'`)
//! and is queried through the engine's existing grant paths;
//! `list_for_subjects` below resolves `role_grants` → `storage.drives`
//! `list_readable_by` below resolves `role_grants` → `storage.drives`
//! via a single join.
//!
//! See `migrations/20260802000000_drives_schema_additive.sql` for the
//! schema and `docs/plan/drive.md` §3 / §15 for the locked design.
use std::sync::Arc;
use std::time::Duration;
use moka::future::Cache;
use sqlx::{PgPool, Row, types::Uuid};
use crate::domain::entities::drive::{Drive, DriveKind};
@@ -18,13 +20,108 @@ use crate::domain::repositories::drive_repository::{
DriveRepository, DriveRepositoryError, DriveWithRootName,
};
/// Decode a `d.policies` JSONB column straight into `DrivePolicies` via
/// `sqlx::types::Json<T>` — a single `serde_json::from_slice` over the raw JSONB
/// bytes — instead of fetching a throwaway `serde_json::Value` DOM and walking it
/// once with `DrivePolicies::from_value`. The §J1 pattern (ROUND23) applied to
/// the drive-policy path §J2 left behind (benches/ROUND26.md §P1). The lenient
/// `unwrap_or_default` fallback (a malformed bag decodes to all-false rather than
/// erroring the read) is preserved exactly.
fn policies_from_row(row: &sqlx::postgres::PgRow) -> crate::domain::entities::drive::DrivePolicies {
row.try_get::<sqlx::types::Json<crate::domain::entities::drive::DrivePolicies>, _>("policies")
.map(|j| j.0)
.unwrap_or_default()
}
/// `default_drive_cache` TTL. The default-drive → root-folder binding is
/// nearly immutable (changes only on provisioning / drive deletion /
/// policy edits — all of which invalidate explicitly below), yet it is
/// re-resolved on EVERY NextCloud request (basic-auth chroot), every
/// native `/webdav` request (Mode-B scope resolution) and every WOPI
/// call. 30 s mirrors `drive_role_cache` in `pg_acl_engine.rs`. Root-
/// folder renames — which don't pass through this repository directly
/// — invalidate via the `DriveRepository::invalidate_default_drive_all`
/// trait hook called from `folder_service::rename_folder_with_perms`
/// when `parent_id IS NULL`. Measured in `benches/CHROOT-CACHE.md`.
const DEFAULT_DRIVE_CACHE_TTL: Duration = Duration::from_secs(30);
/// One entry per active user; entries are small (a `Drive` + a name).
const DEFAULT_DRIVE_CACHE_CAPACITY: u64 = 100_000;
pub struct DrivePgRepository {
pool: Arc<PgPool>,
/// user_id → default drive (+ root folder name). See
/// [`DEFAULT_DRIVE_CACHE_TTL`]. Only `Ok` results are cached, so the
/// provisioning idempotency check (`NotFound` → create) always sees
/// the live table.
default_drive_cache: Cache<Uuid, DriveWithRootName>,
/// caller_id → every drive the caller can read (the full
/// role_grants ⋈ drives ⋈ folders join of [`list_readable_by`],
/// including the transitive-group expansion).
///
/// Re-resolved before this cache existed on EVERY native `/webdav`
/// request that names an explicit drive selector (all verbs; MOVE
/// and COPY twice), plus per-request in search, trash listing and
/// the `GET /api/drives` picker — the heaviest per-request query
/// left on the DAV path after CHROOT-CACHE. Concurrent misses are
/// coalesced (`try_get_with`), errors are never cached.
///
/// Freshness: every membership/lifecycle mutation that flows
/// through this repository or `DriveManagementService` invalidates
/// explicitly (per-user when the subject is a User, whole cache for
/// Group subjects, whose transitive membership is not resolvable
/// here). Root-folder renames — which update `drive.name` because it
/// reads through `folders.name` of the root row — also invalidate,
/// via the trait's `invalidate_readable_all` hook called from
/// `folder_service::rename_folder_with_perms` when
/// `parent_id IS NULL`. That path was missed by the perf commit
/// that introduced this cache (`12dc648c`) and surfaced by
/// `drives_membership.hurl` Step 23; the trait hook closes it
/// without folder_service knowing about the concrete moka cache.
///
/// Residual staleness — a grant written by a path that can't reach
/// this cache — is bounded by the same 30 s TTL the sibling caches
/// accept; actual permission enforcement is unaffected (the ACL
/// engine re-checks per operation with its own invalidation).
readable_cache: Cache<Uuid, Arc<Vec<DriveWithRootName>>>,
}
impl DrivePgRepository {
pub fn new(pool: Arc<PgPool>) -> Self {
Self { pool }
Self {
pool,
default_drive_cache: Cache::builder()
.max_capacity(DEFAULT_DRIVE_CACHE_CAPACITY)
.time_to_live(DEFAULT_DRIVE_CACHE_TTL)
.build(),
readable_cache: Cache::builder()
.max_capacity(DEFAULT_DRIVE_CACHE_CAPACITY)
.time_to_live(DEFAULT_DRIVE_CACHE_TTL)
.build(),
}
}
/// Drop the cached readable-drive list for one user (their grant set
/// changed: membership write, personal-drive provisioning, …).
pub async fn invalidate_readable_for_user(&self, user_id: Uuid) {
self.readable_cache.invalidate(&user_id).await;
}
/// Drop every cached readable-drive list. Used when the affected
/// user set is unknown at this layer: group-subject grants, drive
/// deletion, policy edits. All are admin-rare; repopulation costs
/// one join per active caller.
pub fn invalidate_readable_all(&self) {
self.readable_cache.invalidate_all();
}
/// Drop every cached `default_drive_cache` entry. Exposed as a
/// `pub` sibling of the whole-cache invalidators above so trait
/// callers holding a `dyn DriveRepository` can trigger the same
/// cleanup path (e.g. `folder_service` on root-folder rename —
/// see `impl DriveRepository` below).
pub fn invalidate_default_drive_all(&self) {
self.default_drive_cache.invalidate_all();
}
fn map_sqlx_err(context: &'static str, e: sqlx::Error) -> DriveRepositoryError {
@@ -75,7 +172,7 @@ impl DrivePgRepository {
/// is declared owner→viewer (strongest→weakest), so `MIN` picks the
/// strongest of the caller's grants on the drive (direct +
/// group-mediated collapsed by GROUP BY). Used only by
/// `list_for_subjects`.
/// `list_readable_by`.
fn row_to_drive_with_name_and_role(
row: &sqlx::postgres::PgRow,
) -> Result<DriveWithRootName, DriveRepositoryError> {
@@ -85,10 +182,83 @@ impl DrivePgRepository {
dwr.caller_role = role_str.as_deref().and_then(Role::parse);
Ok(dwr)
}
/// The uncached grants join behind [`DriveRepository::list_readable_by`].
///
/// Joining role_grants → drives → folders returns every drive the
/// caller can read, paired with its display name. Group
/// memberships (direct + transitive) are expanded inline by
/// `storage.caller_group_ids($caller)` — no Rust-side ceremony.
///
/// ORDER BY puts default drives first (so the picker UI doesn't
/// need a follow-up sort), then alphabetical by name. GROUP BY
/// collapses duplicate role_grants on the same drive (direct +
/// group-mediated) and sidesteps PostgreSQL's "ORDER BY
/// expression must appear in select list" rule that SELECT
/// DISTINCT imposes.
/// `MIN(g.role)` picks the caller's strongest role on each drive:
/// `storage.grant_role` is declared `owner → viewer` (strongest →
/// weakest), so MIN returns the strongest. Cast `::text` matches
/// the codebase convention for reading enum columns into Rust
/// (see `pg_acl_engine.rs`); `Role::parse` handles the trip back.
async fn query_readable_by(
&self,
caller_id: Uuid,
) -> Result<Vec<DriveWithRootName>, DriveRepositoryError> {
let rows = sqlx::query(
r#"
SELECT d.id, d.kind, d.default_for_user, d.root_folder_id,
d.quota_bytes, d.used_bytes, d.policies,
d.created_at, d.updated_at,
f.name AS root_folder_name,
MIN(g.role)::text AS caller_role
FROM storage.drives d
JOIN storage.folders f ON f.id = d.root_folder_id
JOIN storage.role_grants g
ON g.resource_type = 'drive'
AND g.resource_id = d.id
WHERE (
(g.subject_type = 'user' AND g.subject_id = $1)
OR (g.subject_type = 'group' AND g.subject_id IN
(SELECT storage.caller_group_ids($1)))
)
AND (g.expires_at IS NULL OR g.expires_at > NOW())
GROUP BY d.id, d.kind, d.default_for_user, d.root_folder_id,
d.quota_bytes, d.used_bytes, d.policies,
d.created_at, d.updated_at, f.name
ORDER BY (d.default_for_user IS NULL) ASC,
LOWER(f.name) ASC
"#,
)
.bind(caller_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("list_readable_by", e))?;
rows.iter()
.map(Self::row_to_drive_with_name_and_role)
.collect()
}
}
#[async_trait::async_trait]
impl DriveRepository for DrivePgRepository {
async fn invalidate_readable_for_user(&self, user_id: Uuid) {
// Delegate to the inherent method — the trait forwarding lets
// callers holding a `dyn DriveRepository` (e.g. `folder_service`
// on a root-folder rename) trigger invalidation without knowing
// about the concrete cache.
DrivePgRepository::invalidate_readable_for_user(self, user_id).await;
}
fn invalidate_readable_all(&self) {
DrivePgRepository::invalidate_readable_all(self);
}
fn invalidate_default_drive_all(&self) {
DrivePgRepository::invalidate_default_drive_all(self);
}
async fn create_personal_drive_atomic(
&self,
owner_id: Uuid,
@@ -116,11 +286,22 @@ impl DriveRepository for DrivePgRepository {
.map_err(|e| Self::map_sqlx_err("create_personal_drive_atomic.begin", e))?;
// 1. Drive row (root_folder_id NULL — populated in step 3).
//
// Default personal drives are seeded with `include_in_photo_index`
// + `include_in_music_index` = true so the Photos / Music
// predicates (§15) can be a single positive rule keyed off the
// JSONB flag — no per-kind carve-out needed at query time. Any
// future admin PATCH toggling either flag off shows a confirm
// dialog in the UI (unusual action; empties the user's Photos
// timeline / Music library).
let drive_id: Uuid = sqlx::query_scalar(
r#"
INSERT INTO storage.drives
(kind, default_for_user, quota_bytes, policies)
VALUES ('personal', $1, $2, '{}'::jsonb)
VALUES (
'personal', $1, $2,
'{"include_in_photo_index": true, "include_in_music_index": true}'::jsonb
)
RETURNING id
"#,
)
@@ -132,11 +313,16 @@ impl DriveRepository for DrivePgRepository {
// 2. Root folder. `parent_id IS NULL` makes it a root in the
// drive; `drive_id` closes the FK in this direction.
//
// Post-D7: `user_id` omitted from the INSERT column list —
// the column is nullable and no longer written to on new
// rows. `created_by` / `updated_by` bind to the owner
// (§14 provenance).
let folder_id: Uuid = sqlx::query_scalar(
r#"
INSERT INTO storage.folders
(name, parent_id, user_id, drive_id, created_by, updated_by)
VALUES ('Personal', NULL, $1, $2, $1, $1)
(name, parent_id, drive_id, created_by, updated_by)
VALUES ('Personal', NULL, $2, $1, $1)
RETURNING id
"#,
)
@@ -193,6 +379,12 @@ impl DriveRepository for DrivePgRepository {
.await
.map_err(|e| Self::map_sqlx_err("create_personal_drive_atomic.commit", e))?;
// Drop any cached default-drive resolution for this user (a stale
// NotFound is never cached, but be explicit about the write path).
self.default_drive_cache.invalidate(&owner_id).await;
// The owner gained a drive — their readable list changed too.
self.invalidate_readable_for_user(owner_id).await;
Self::row_to_drive_with_name(&row)
}
@@ -233,14 +425,14 @@ impl DriveRepository for DrivePgRepository {
.await
.map_err(|e| Self::map_sqlx_err("create_shared_drive_atomic.drive", e))?;
// 2. Root folder. The folder's `user_id` carries the admin (legacy
// column still NOT NULL during the dual-write window — D7
// drops it once `drive_id` is the canonical ownership signal).
// 2. Root folder. Post-D7: `user_id` omitted — the column is
// nullable and unused on new rows. `created_by` / `updated_by`
// bind to `granted_by` (§14 provenance).
let folder_id: Uuid = sqlx::query_scalar(
r#"
INSERT INTO storage.folders
(name, parent_id, user_id, drive_id, created_by, updated_by)
VALUES ($1, NULL, $2, $3, $2, $2)
(name, parent_id, drive_id, created_by, updated_by)
VALUES ($1, NULL, $3, $2, $2)
RETURNING id
"#,
)
@@ -300,9 +492,107 @@ impl DriveRepository for DrivePgRepository {
.await
.map_err(|e| Self::map_sqlx_err("create_shared_drive_atomic.commit", e))?;
// The owner grant written above changes the grantee's readable
// list. User subjects invalidate precisely; Group subjects fall
// back to a full clear (transitive members unknown here).
match owner_subject {
crate::domain::services::authorization::Subject::User(uid) => {
self.invalidate_readable_for_user(uid).await;
}
_ => self.invalidate_readable_all(),
}
Self::row_to_drive_with_name(&row)
}
async fn is_empty(&self, drive_id: Uuid) -> Result<bool, DriveRepositoryError> {
// A "live" non-root folder = any folder with `parent_id IS NOT
// NULL` (root is the only NULL-parent row per drive) and not in
// the trash. Trashed items don't count — owners can delete a
// drive even when its trash bin still holds rows; the trash GC
// will clean those up after the standard retention window.
//
// EXISTS instead of COUNT(*): only emptiness is tested, so the
// planner stops at the first matching row — a populated drive
// answers from one index probe instead of aggregating every
// live file + folder it contains.
let occupied: (bool,) = sqlx::query_as(
r#"
SELECT EXISTS(
SELECT 1 FROM storage.folders
WHERE drive_id = $1 AND parent_id IS NOT NULL AND NOT is_trashed)
OR EXISTS(
SELECT 1 FROM storage.files
WHERE drive_id = $1 AND NOT is_trashed)
"#,
)
.bind(drive_id)
.fetch_one(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("is_empty", e))?;
Ok(!occupied.0)
}
async fn delete_atomic(&self, drive_id: Uuid) -> Result<(), DriveRepositoryError> {
// Three-statement transaction:
// 1. Drop every role_grants row scoped to the drive itself
// (folder/file grants under it are gone by step 3 cascade).
// 2. Look up the root folder id (we'll need it to delete the
// folder row AFTER the drive row releases its FK).
// 3. Delete the drive — release the drive→root FK first.
// 4. Delete the root folder (drive_id FK on folders cascades
// from this row going away; only the root remains because
// is_empty was true).
//
// `drive_id` is bound once per statement; failure at any step
// rolls back. Caller (`DriveManagementService::delete_drive`)
// is responsible for the `is_empty` precheck.
let mut tx = self
.pool
.begin()
.await
.map_err(|e| Self::map_sqlx_err("delete_atomic.begin", e))?;
sqlx::query(
"DELETE FROM storage.role_grants \
WHERE resource_type = 'drive' AND resource_id = $1",
)
.bind(drive_id)
.execute(&mut *tx)
.await
.map_err(|e| Self::map_sqlx_err("delete_atomic.grants", e))?;
let root: (Uuid,) =
sqlx::query_as("SELECT root_folder_id FROM storage.drives WHERE id = $1")
.bind(drive_id)
.fetch_optional(&mut *tx)
.await
.map_err(|e| Self::map_sqlx_err("delete_atomic.lookup_root", e))?
.ok_or_else(|| DriveRepositoryError::NotFound(drive_id.to_string()))?;
sqlx::query("DELETE FROM storage.drives WHERE id = $1")
.bind(drive_id)
.execute(&mut *tx)
.await
.map_err(|e| Self::map_sqlx_err("delete_atomic.drive", e))?;
sqlx::query("DELETE FROM storage.folders WHERE id = $1")
.bind(root.0)
.execute(&mut *tx)
.await
.map_err(|e| Self::map_sqlx_err("delete_atomic.root", e))?;
tx.commit()
.await
.map_err(|e| Self::map_sqlx_err("delete_atomic.commit", e))?;
// We only have the drive id here; the caches are keyed by user.
// Deletion is rare — clearing them whole is the simple,
// always-correct move (repopulates at one query per active user).
self.default_drive_cache.invalidate_all();
self.invalidate_readable_all();
Ok(())
}
async fn get_by_id(&self, id: Uuid) -> Result<DriveWithRootName, DriveRepositoryError> {
let row = sqlx::query(
r#"
@@ -354,6 +644,10 @@ impl DriveRepository for DrivePgRepository {
&self,
user_id: Uuid,
) -> Result<DriveWithRootName, DriveRepositoryError> {
if let Some(cached) = self.default_drive_cache.get(&user_id).await {
return Ok(cached);
}
let row = sqlx::query(
r#"
SELECT d.id, d.kind, d.default_for_user, d.root_folder_id,
@@ -371,65 +665,30 @@ impl DriveRepository for DrivePgRepository {
.map_err(|e| Self::map_sqlx_err("find_default_for_user", e))?
.ok_or_else(|| DriveRepositoryError::NotFound(user_id.to_string()))?;
Self::row_to_drive_with_name(&row)
let dwr = Self::row_to_drive_with_name(&row)?;
self.default_drive_cache.insert(user_id, dwr.clone()).await;
Ok(dwr)
}
async fn list_for_subjects(
async fn list_readable_by(
&self,
subject_types: &[&str],
subject_ids: &[Uuid],
) -> Result<Vec<DriveWithRootName>, DriveRepositoryError> {
// Joining role_grants → drives → folders returns every drive the
// expanded subject set can read, paired with its display name.
// ORDER BY puts default drives first (so the picker UI doesn't
// need a follow-up sort), then alphabetical by name. GROUP BY
// collapses duplicate role_grants on the same drive (direct +
// group-mediated) and sidesteps PostgreSQL's "ORDER BY
// expression must appear in select list" rule that SELECT
// DISTINCT imposes.
// `MIN(g.role)` picks the caller's strongest role on each drive:
// `storage.grant_role` is declared `owner → viewer` (strongest →
// weakest), so MIN returns the strongest. Cast `::text` matches
// the codebase convention for reading enum columns into Rust
// (see `pg_acl_engine.rs`); `Role::parse` handles the trip back.
// Collapses direct + group-mediated grants on the same drive
// into one row alongside the existing GROUP BY.
let rows = sqlx::query(
r#"
SELECT d.id, d.kind, d.default_for_user, d.root_folder_id,
d.quota_bytes, d.used_bytes, d.policies,
d.created_at, d.updated_at,
f.name AS root_folder_name,
MIN(g.role)::text AS caller_role
FROM storage.drives d
JOIN storage.folders f ON f.id = d.root_folder_id
JOIN storage.role_grants g
ON g.resource_type = 'drive'
AND g.resource_id = d.id
WHERE g.subject_type = ANY($1)
AND g.subject_id = ANY($2)
AND (g.expires_at IS NULL OR g.expires_at > NOW())
GROUP BY d.id, d.kind, d.default_for_user, d.root_folder_id,
d.quota_bytes, d.used_bytes, d.policies,
d.created_at, d.updated_at, f.name
ORDER BY (d.default_for_user IS NULL) ASC,
LOWER(f.name) ASC
"#,
)
.bind(
subject_types
.iter()
.map(|s| s.to_string())
.collect::<Vec<_>>(),
)
.bind(subject_ids)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("list_for_subjects", e))?;
rows.iter()
.map(Self::row_to_drive_with_name_and_role)
.collect()
caller_id: Uuid,
) -> Result<Arc<Vec<DriveWithRootName>>, DriveRepositoryError> {
// Serve from the per-user cache; concurrent misses for the same
// caller are coalesced into one join (`try_get_with`), and errors
// are never cached. See the `readable_cache` field docs for the
// freshness/invalidation contract. The Arc is handed to callers
// directly — a warm hit is a refcount bump, not a deep clone of
// every row's Strings.
self.readable_cache
.try_get_with(caller_id, async move {
self.query_readable_by(caller_id).await.map(Arc::new)
})
.await
.map_err(|e: Arc<DriveRepositoryError>| {
Arc::try_unwrap(e)
.unwrap_or_else(|shared| DriveRepositoryError::StorageError(shared.to_string()))
})
}
async fn list_all(&self) -> Result<Vec<DriveWithRootName>, DriveRepositoryError> {
@@ -455,4 +714,166 @@ impl DriveRepository for DrivePgRepository {
rows.iter().map(Self::row_to_drive_with_name).collect()
}
async fn get_policies_for_file(
&self,
file_id: Uuid,
) -> Result<crate::domain::entities::drive::DrivePolicies, DriveRepositoryError> {
let row = sqlx::query(
"SELECT d.policies \
FROM storage.drives d \
JOIN storage.files f ON f.drive_id = d.id \
WHERE f.id = $1",
)
.bind(file_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("get_policies_for_file", e))?
.ok_or_else(|| DriveRepositoryError::NotFound(file_id.to_string()))?;
Ok(policies_from_row(&row))
}
async fn get_policies_for_folder(
&self,
folder_id: Uuid,
) -> Result<crate::domain::entities::drive::DrivePolicies, DriveRepositoryError> {
let row = sqlx::query(
"SELECT d.policies \
FROM storage.drives d \
JOIN storage.folders fo ON fo.drive_id = d.id \
WHERE fo.id = $1",
)
.bind(folder_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("get_policies_for_folder", e))?
.ok_or_else(|| DriveRepositoryError::NotFound(folder_id.to_string()))?;
Ok(policies_from_row(&row))
}
async fn get_drive_id_and_policies_for_file(
&self,
file_id: Uuid,
) -> Result<(Uuid, crate::domain::entities::drive::DrivePolicies), DriveRepositoryError> {
let row = sqlx::query(
"SELECT d.id, d.policies \
FROM storage.drives d \
JOIN storage.files f ON f.drive_id = d.id \
WHERE f.id = $1",
)
.bind(file_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("get_drive_id_and_policies_for_file", e))?
.ok_or_else(|| DriveRepositoryError::NotFound(file_id.to_string()))?;
let drive_id: Uuid = row
.try_get("id")
.map_err(|e| Self::map_sqlx_err("get_drive_id_and_policies_for_file", e))?;
Ok((drive_id, policies_from_row(&row)))
}
async fn get_drive_id_and_policies_for_folder(
&self,
folder_id: Uuid,
) -> Result<(Uuid, crate::domain::entities::drive::DrivePolicies), DriveRepositoryError> {
let row = sqlx::query(
"SELECT d.id, d.policies \
FROM storage.drives d \
JOIN storage.folders fo ON fo.drive_id = d.id \
WHERE fo.id = $1",
)
.bind(folder_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("get_drive_id_and_policies_for_folder", e))?
.ok_or_else(|| DriveRepositoryError::NotFound(folder_id.to_string()))?;
let drive_id: Uuid = row
.try_get("id")
.map_err(|e| Self::map_sqlx_err("get_drive_id_and_policies_for_folder", e))?;
Ok((drive_id, policies_from_row(&row)))
}
async fn drive_id_for_folder(&self, folder_id: Uuid) -> Result<Uuid, DriveRepositoryError> {
let row: Option<(Uuid,)> =
sqlx::query_as("SELECT drive_id FROM storage.folders WHERE id = $1")
.bind(folder_id)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("drive_id_for_folder", e))?;
row.map(|(id,)| id)
.ok_or_else(|| DriveRepositoryError::NotFound(folder_id.to_string()))
}
async fn update_policies(
&self,
drive_id: Uuid,
partial: &serde_json::Value,
) -> Result<crate::domain::entities::drive::DrivePolicies, DriveRepositoryError> {
// JSONB-level merge (`||`) keeps unknown keys already on disk —
// the column remains the canonical bag (see
// `DrivePolicies::from_value` — typed read is lenient, untyped
// write is preserving). The caller passes a raw `Value` with
// ONLY the keys it wants to change (never a full `DrivePolicies`
// round-trip, which would serialise all-false defaults into the
// merge and clobber other flags). RETURNING surfaces the
// post-merge bag so the audit log shows what the row actually
// carries afterwards.
let row: Option<(serde_json::Value,)> = sqlx::query_as(
"UPDATE storage.drives \
SET policies = policies || $2, \
updated_at = now() \
WHERE id = $1 \
RETURNING policies",
)
.bind(drive_id)
.bind(partial)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("update_policies", e))?;
let raw = row
.ok_or_else(|| DriveRepositoryError::NotFound(drive_id.to_string()))?
.0;
// Policy edits must not serve a stale `policies` bag from the
// user-keyed caches (we only have the drive id) — clear both;
// policy edits are admin-rare.
self.default_drive_cache.invalidate_all();
self.invalidate_readable_all();
Ok(crate::domain::entities::drive::DrivePolicies::from_value(
&raw,
))
}
async fn update_quota(
&self,
drive_id: Uuid,
quota_bytes: Option<i64>,
) -> Result<Option<i64>, DriveRepositoryError> {
// RETURNING gives the persisted value so the caller (service
// layer) has authoritative data for the audit line + API
// response without a second read.
let row: Option<(Option<i64>,)> = sqlx::query_as(
"UPDATE storage.drives \
SET quota_bytes = $2, \
updated_at = now() \
WHERE id = $1 \
RETURNING quota_bytes",
)
.bind(drive_id)
.bind(quota_bytes)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| Self::map_sqlx_err("update_quota", e))?;
let persisted = row
.ok_or_else(|| DriveRepositoryError::NotFound(drive_id.to_string()))?
.0;
// Same invalidation strategy as `update_policies` — both
// user-keyed caches (`default_drive_cache`, the readable-drive
// list) carry the whole DriveWithRootName / DriveDto rows and
// would serve a stale quota otherwise. Admin-rare mutation,
// so blowing the whole cache is fine (no per-user pinpointing
// needed).
self.default_drive_cache.invalidate_all();
self.invalidate_readable_all();
Ok(persisted)
}
}
@@ -13,7 +13,7 @@ use uuid::Uuid;
use crate::application::ports::face_ports::FaceRepository;
use crate::common::errors::DomainError;
use crate::domain::entities::face::{BoundingBox, Face, Person};
use crate::domain::entities::face::{BoundingBox, Face, FaceBox, Person};
/// Row shape for `faces.faces` selects (avoids `clippy::type_complexity`).
type FaceRow = (
@@ -115,40 +115,98 @@ impl FaceRepository for FacePgRepository {
if faces.is_empty() {
return Ok(());
}
let mut tx = self.pool.begin().await.map_err(|e| db_err("begin", e))?;
// One multi-row INSERT over parallel UNNEST arrays instead of one
// round-trip per face — a group photo yields many faces per indexed
// image. The `bbox` float4[] can't ride an array-of-arrays through
// unnest (PG flattens), so its 4 components travel as 4 parallel
// arrays and are reassembled server-side. A single statement is
// atomic on its own; the per-row transaction wrapper is gone.
let n = faces.len();
let mut ids = Vec::with_capacity(n);
let mut file_ids = Vec::with_capacity(n);
let mut user_ids = Vec::with_capacity(n);
let mut person_ids: Vec<Option<Uuid>> = Vec::with_capacity(n);
let (mut bx, mut by, mut bw, mut bh) = (
Vec::with_capacity(n),
Vec::with_capacity(n),
Vec::with_capacity(n),
Vec::with_capacity(n),
);
let mut det_scores = Vec::with_capacity(n);
let mut qualities: Vec<Option<f32>> = Vec::with_capacity(n);
let mut embeddings = Vec::with_capacity(n);
let mut blob_hashes: Vec<Option<&str>> = Vec::with_capacity(n);
for f in faces {
sqlx::query(
r#"
INSERT INTO faces.faces
(id, file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
"#,
)
.bind(f.id)
.bind(f.file_id)
.bind(f.user_id)
.bind(f.person_id)
.bind(f.bbox.to_array())
.bind(f.det_score)
.bind(f.quality)
.bind(embedding_to_bytes(&f.embedding))
.bind(f.blob_hash.as_deref())
.execute(&mut *tx)
.await
.map_err(|e| db_err("save_faces", e))?;
ids.push(f.id);
file_ids.push(f.file_id);
user_ids.push(f.user_id);
person_ids.push(f.person_id);
bx.push(f.bbox.x);
by.push(f.bbox.y);
bw.push(f.bbox.w);
bh.push(f.bbox.h);
det_scores.push(f.det_score);
qualities.push(f.quality);
embeddings.push(embedding_to_bytes(&f.embedding));
blob_hashes.push(f.blob_hash.as_deref());
}
tx.commit().await.map_err(|e| db_err("commit", e))?;
sqlx::query(
r#"
INSERT INTO faces.faces
(id, file_id, user_id, person_id, bbox, det_score, quality, embedding, blob_hash)
SELECT t.id, t.file_id, t.user_id, t.person_id,
ARRAY[t.bx, t.by, t.bw, t.bh]::real[],
t.det_score, t.quality, t.embedding, t.blob_hash
FROM unnest($1::uuid[], $2::uuid[], $3::uuid[], $4::uuid[],
$5::real[], $6::real[], $7::real[], $8::real[],
$9::real[], $10::real[], $11::bytea[], $12::text[])
AS t(id, file_id, user_id, person_id,
bx, by, bw, bh, det_score, quality, embedding, blob_hash)
"#,
)
.bind(&ids)
.bind(&file_ids)
.bind(&user_ids)
.bind(&person_ids)
.bind(&bx)
.bind(&by)
.bind(&bw)
.bind(&bh)
.bind(&det_scores)
.bind(&qualities)
.bind(&embeddings)
.bind(&blob_hashes)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("save_faces", e))?;
Ok(())
}
async fn faces_for_file(&self, file_id: Uuid) -> Result<Vec<Face>, DomainError> {
let sql = format!("SELECT {FACE_COLS} FROM faces.faces WHERE file_id = $1");
let rows: Vec<FaceRow> = sqlx::query_as(&sql)
.bind(file_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("faces_for_file", e))?;
Ok(rows.into_iter().map(row_to_face).collect())
async fn face_boxes_for_file(
&self,
file_id: Uuid,
user_id: Uuid,
) -> Result<Vec<FaceBox>, DomainError> {
// Narrow projection: the lightbox needs only (id, person_id, bbox), so
// the 2 KiB embedding BYTEA + 6 unused columns stay in the DB and the
// caller filter runs in SQL (idx_faces_file drives it) rather than in
// Rust after a full-row fetch. See benches/ROUND14.md §Q1.
let rows: Vec<(Uuid, Option<Uuid>, Vec<f32>)> = sqlx::query_as(
"SELECT id, person_id, bbox FROM faces.faces WHERE file_id = $1 AND user_id = $2",
)
.bind(file_id)
.bind(user_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("face_boxes_for_file", e))?;
Ok(rows
.into_iter()
.map(|(id, person_id, bbox)| FaceBox {
id,
person_id,
bbox: BoundingBox::from_slice(&bbox),
})
.collect())
}
async fn delete_faces_for_file(&self, file_id: Uuid) -> Result<(), DomainError> {
@@ -186,6 +244,60 @@ impl FaceRepository for FacePgRepository {
Ok(rows.into_iter().map(row_to_face).collect())
}
async fn person_face_stats(&self, user_id: Uuid) -> Result<Vec<(Uuid, i64)>, DomainError> {
// Grouped COUNT — the People tab only needs per-person counts, so
// this replaces a full faces_for_user scan that shipped a 2 KiB
// embedding BYTEA per row (benches/PEOPLE-LIST.md).
let rows: Vec<(Uuid, i64)> = sqlx::query_as(
"SELECT person_id, COUNT(*) FROM faces.faces
WHERE user_id = $1 AND person_id IS NOT NULL
GROUP BY person_id",
)
.bind(user_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("person_face_stats", e))?;
Ok(rows)
}
async fn file_ids_for_faces(
&self,
user_id: Uuid,
face_ids: &[Uuid],
) -> Result<std::collections::HashMap<Uuid, Uuid>, DomainError> {
if face_ids.is_empty() {
return Ok(std::collections::HashMap::new());
}
let rows: Vec<(Uuid, Uuid)> = sqlx::query_as(
"SELECT id, file_id FROM faces.faces WHERE user_id = $1 AND id = ANY($2)",
)
.bind(user_id)
.bind(face_ids)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| db_err("file_ids_for_faces", e))?;
Ok(rows.into_iter().collect())
}
async fn reassign_person_faces(
&self,
user_id: Uuid,
from: Uuid,
into: Uuid,
) -> Result<u64, DomainError> {
let result = sqlx::query(
"UPDATE faces.faces SET person_id = $3
WHERE user_id = $1 AND person_id = $2",
)
.bind(user_id)
.bind(from)
.bind(into)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("reassign_person_faces", e))?;
Ok(result.rows_affected())
}
async fn assign_person(
&self,
face_id: Uuid,
@@ -200,6 +312,28 @@ impl FaceRepository for FacePgRepository {
Ok(())
}
async fn assign_person_batch(
&self,
assignments: &[(Uuid, Option<Uuid>)],
) -> Result<(), DomainError> {
if assignments.is_empty() {
return Ok(());
}
let (face_ids, person_ids): (Vec<Uuid>, Vec<Option<Uuid>>) =
assignments.iter().cloned().unzip();
sqlx::query(
"UPDATE faces.faces f SET person_id = u.pid
FROM (SELECT unnest($1::uuid[]) AS fid, unnest($2::uuid[]) AS pid) u
WHERE f.id = u.fid",
)
.bind(&face_ids)
.bind(&person_ids)
.execute(self.pool.as_ref())
.await
.map_err(|e| db_err("assign_person_batch", e))?;
Ok(())
}
async fn create_person(&self, person: &Person) -> Result<(), DomainError> {
sqlx::query(
r#"
@@ -24,25 +24,27 @@ impl FavoritesPgRepository {
impl FavoritesRepositoryPort for FavoritesPgRepository {
async fn get_favorites(&self, user_id: Uuid) -> Result<Vec<FavoriteItemDto>> {
// `id`/`user_id`/`parent_id` decode as binary UUIDs (16 B on the wire,
// no server-side `::TEXT` cast) and render app-side — the ROUND6 §10
// pattern the two legacy listing methods here never picked up.
let rows = sqlx::query(
r#"
SELECT
uf.id::TEXT AS "id",
uf.user_id::TEXT AS "user_id",
uf.id AS "id",
uf.user_id AS "user_id",
uf.item_id AS "item_id",
uf.item_type AS "item_type",
uf.created_at AS "created_at",
COALESCE(f.name, fld.name) AS "item_name",
f.size AS "item_size",
f.mime_type AS "item_mime_type",
COALESCE(f.folder_id::TEXT, fld.parent_id::TEXT) AS "parent_id",
COALESCE(f.folder_id, fld.parent_id) AS "parent_id",
COALESCE(f.updated_at, fld.updated_at) AS "modified_at",
CASE
WHEN uf.item_type = 'folder' THEN fld.path
WHEN uf.item_type = 'file' THEN COALESCE(pfld.path || '/' || f.name, f.name)
ELSE NULL
END AS "item_path",
COALESCE(f.user_id, fld.user_id)::TEXT AS "owner_id"
END AS "item_path"
FROM auth.user_favorites uf
LEFT JOIN storage.files f ON uf.item_type = 'file'
AND f.id = uf.item_id::UUID
@@ -71,18 +73,21 @@ impl FavoritesRepositoryPort for FavoritesPgRepository {
.iter()
.map(|row| {
FavoriteItemDto {
id: row.get("id"),
user_id: row.get("user_id"),
id: row.get::<i32, _>("id").to_string(),
user_id: row.get::<Uuid, _>("user_id").to_string(),
item_id: row.get("item_id"),
item_type: row.get("item_type"),
created_at: row.get("created_at"),
item_name: row.try_get("item_name").ok(),
item_size: row.try_get("item_size").ok(),
item_mime_type: row.try_get("item_mime_type").ok(),
parent_id: row.try_get("parent_id").ok(),
parent_id: row
.try_get::<Option<Uuid>, _>("parent_id")
.ok()
.flatten()
.map(|u| u.to_string()),
modified_at: row.try_get("modified_at").ok(),
item_path: row.try_get("item_path").ok(),
owner_id: row.try_get("owner_id").ok(),
// Temporary defaults; with_display_fields() computes the real values
icon_class: String::new(),
icon_special_class: String::new(),
@@ -261,8 +266,9 @@ impl FavoritesRepositoryPort for FavoritesPgRepository {
return Ok(HashSet::new());
}
// Collect just the IDs for the IN clause
let ids: Vec<String> = item_ids.iter().map(|(id, _)| id.to_string()).collect();
// Collect just the IDs for the IN clause — sqlx binds `&[&str]` as
// text[], so no per-id String is needed.
let ids: Vec<&str> = item_ids.iter().map(|(id, _)| *id).collect();
let rows = sqlx::query(
"SELECT item_id FROM auth.user_favorites WHERE user_id = $1 AND item_id = ANY($2)",
@@ -309,9 +315,19 @@ impl FavoritesRepositoryPort for FavoritesPgRepository {
-1::bigint AS size,
fld.created_at AS resource_created_at,
fld.updated_at AS modified_at,
fld.user_id AS owner_id,
fld.drive_id AS drive_id,
NULL::text AS blob_hash,
(fld.user_id = $1::uuid) AS is_owner,
fld.created_by AS created_by,
fld.updated_by AS updated_by,
EXISTS (
SELECT 1 FROM storage.role_grants g
WHERE g.resource_type = 'drive'
AND g.resource_id = fld.drive_id
AND g.role = 'owner'
AND g.subject_type = 'user'
AND g.subject_id = $1::uuid
AND (g.expires_at IS NULL OR g.expires_at > NOW())
) AS is_owner,
uf.created_at AS favorited_at,
fld.path::text AS resource_path,
LOWER(fld.name) AS sort_str,
@@ -332,9 +348,19 @@ impl FavoritesRepositoryPort for FavoritesPgRepository {
f.size::bigint,
f.created_at AS resource_created_at,
f.updated_at AS modified_at,
f.user_id AS owner_id,
f.drive_id AS drive_id,
f.blob_hash,
(f.user_id = $1::uuid) AS is_owner,
f.created_by AS created_by,
f.updated_by AS updated_by,
EXISTS (
SELECT 1 FROM storage.role_grants g
WHERE g.resource_type = 'drive'
AND g.resource_id = f.drive_id
AND g.role = 'owner'
AND g.subject_type = 'user'
AND g.subject_id = $1::uuid
AND (g.expires_at IS NULL OR g.expires_at > NOW())
) AS is_owner,
uf.created_at AS favorited_at,
COALESCE(pfld.path::text || '/' || f.name, f.name) AS resource_path,
LOWER(f.name) AS sort_str,
@@ -487,7 +513,8 @@ impl FavoritesRepositoryPort for FavoritesPgRepository {
};
let user_join = if need_user_join {
"LEFT JOIN auth.users u ON u.id = r.owner_id"
// Post-D7: `owner_id` retired; join by `created_by`.
"LEFT JOIN auth.users u ON u.id = r.created_by"
} else {
""
};
@@ -504,7 +531,8 @@ impl FavoritesRepositoryPort for FavoritesPgRepository {
SELECT
r.resource_type, r.resource_id, r.name, r.parent_id,
r.mime_type, r.size, r.resource_created_at, r.modified_at,
r.owner_id, r.is_owner, r.favorited_at, r.resource_path,
r.drive_id, r.blob_hash, r.created_by, r.updated_by,
r.is_owner, r.favorited_at, r.resource_path,
r.sort_str, r.type_order, r.folder_first{username_col}
FROM resources r
{user_join}
@@ -575,8 +603,10 @@ LIMIT $6"
size,
resource_created_at: row.get("resource_created_at"),
modified_at: row.get("modified_at"),
owner_id: row.get("owner_id"),
drive_id: row.get("drive_id"),
blob_hash: row.try_get("blob_hash").ok(),
created_by: row.try_get("created_by").ok(),
updated_by: row.try_get("updated_by").ok(),
is_owner: row.try_get("is_owner").unwrap_or(false),
favorited_at: row.get("favorited_at"),
path: row.try_get("resource_path").ok(),
File diff suppressed because it is too large Load Diff
@@ -17,7 +17,6 @@ use crate::application::dtos::display_helpers::category_order_for;
use crate::application::ports::storage_ports::{CopyFolderTreeResult, FileWritePort};
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::services::path_service::StoragePath;
use super::transaction_utils::retry_on_deadlock;
use crate::infrastructure::services::dedup_service::DedupService;
@@ -61,14 +60,6 @@ impl FileBlobWriteRepository {
}
}
/// Build a `StoragePath` from the materialized folder path + file name.
fn make_file_path(folder_path: Option<&str>, file_name: &str) -> StoragePath {
match folder_path {
Some(fp) if !fp.is_empty() => StoragePath::from_string(&format!("{fp}/{file_name}")),
_ => StoragePath::from_string(file_name),
}
}
/// Look up the materialized folder path. O(1) — no recursive CTE.
async fn lookup_folder_path(
&self,
@@ -104,22 +95,19 @@ impl FileBlobWriteRepository {
mime_type: String,
created_at: i64,
modified_at: i64,
owner_id: Option<Uuid>,
blob_hash: String,
created_by: Option<Uuid>,
updated_by: Option<Uuid>,
) -> Result<File, DomainError> {
let storage_path = Self::make_file_path(folder_path.as_deref(), &name);
File::with_timestamps_blob_hash_and_provenance(
File::from_materialized_row(
id,
name,
storage_path,
folder_path.as_deref(),
size as u64,
mime_type,
folder_id,
created_at as u64,
modified_at as u64,
owner_id,
blob_hash,
created_by,
updated_by,
@@ -127,30 +115,24 @@ impl FileBlobWriteRepository {
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("entity: {e}")))
}
/// Derive `(user_id, drive_id)` from the parent folder. Both are
/// needed during the D0 dual-write window: `user_id` for the legacy
/// column (dropped in D7) and `drive_id` for the new owning-drive
/// reference.
async fn resolve_owner_and_drive(
&self,
folder_id: Option<&str>,
) -> Result<(Uuid, Uuid), DomainError> {
/// Derive `drive_id` from the parent folder. Post-D7: only the
/// drive is needed — the legacy `user_id` column is no longer
/// written on new rows.
async fn resolve_parent_drive(&self, folder_id: Option<&str>) -> Result<Uuid, DomainError> {
match folder_id {
Some(fid) => {
let row: Option<(Uuid, Uuid)> = sqlx::query_as::<_, (Uuid, Uuid)>(
"SELECT user_id, drive_id FROM storage.folders WHERE id = $1::uuid",
)
.bind(fid)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("FileBlobWrite", format!("parent lookup: {e}"))
})?;
row.ok_or_else(|| DomainError::not_found("Folder", fid))
}
Some(fid) => sqlx::query_scalar::<_, Uuid>(
"SELECT drive_id FROM storage.folders WHERE id = $1::uuid",
)
.bind(fid)
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("FileBlobWrite", format!("parent lookup: {e}"))
})?
.ok_or_else(|| DomainError::not_found("Folder", fid)),
None => Err(DomainError::internal_error(
"FileBlobWrite",
"folder_id is required to determine file owner",
"folder_id is required to determine the target drive",
)),
}
}
@@ -171,6 +153,15 @@ impl FileBlobWriteRepository {
/// row — not the row's owner. D2 shared drives let non-owners
/// overwrite content; the previous `updated_by = f.user_id` would
/// have silently recorded the wrong principal.
/// `expected_hash`, when `Some`, turns this into a real
/// compare-and-swap: the SET clause only takes effect if the row's
/// `blob_hash` still matches at the moment the `FOR UPDATE` lock is
/// held (same statement, same transaction — no gap a concurrent
/// writer can land in). A mismatch leaves the row untouched and is
/// reported back via the `matched` flag rather than silently
/// overwriting a sibling PATCH's content. `None` preserves the old
/// blind-overwrite behaviour for PUT/WOPI/chunked-upload finalize,
/// where last-write-wins is the intended HTTP semantics.
async fn swap_blob_hash(
&self,
file_id: &str,
@@ -178,57 +169,83 @@ impl FileBlobWriteRepository {
new_size: i64,
modified_at: Option<i64>,
caller_id: Uuid,
expected_hash: Option<&str>,
) -> Result<(String, i64), DomainError> {
// Atomic CTE: capture old hash then update in one round-trip, no TOCTOU.
// Atomic CTE: capture old hash then conditionally update in one
// round-trip, no TOCTOU. The CASE arms make the SET a no-op when
// `expected_hash` is given and doesn't match `old.blob_hash` —
// the row is still returned (with its unchanged values) so the
// caller can tell "mismatch" apart from "file not found".
// Deadlock victims (40P01) retry before the compensation below runs —
// a successful retry must keep the new blob reference alive.
let (old_hash, updated_at) = match retry_on_deadlock("files.swap_blob_hash", || {
sqlx::query_as::<_, (String, i64)>(
r#"
let (old_hash, updated_at, matched) =
match retry_on_deadlock("files.swap_blob_hash", || {
sqlx::query_as::<_, (String, i64, bool)>(
r#"
WITH old AS (
SELECT id, blob_hash FROM storage.files WHERE id = $3::uuid FOR UPDATE
)
UPDATE storage.files f
SET blob_hash = $1, size = $2,
updated_at = COALESCE(to_timestamp($4), NOW()),
updated_by = $5
SET blob_hash = CASE WHEN $6::text IS NULL OR old.blob_hash = $6
THEN $1 ELSE f.blob_hash END,
size = CASE WHEN $6::text IS NULL OR old.blob_hash = $6
THEN $2 ELSE f.size END,
updated_at = CASE WHEN $6::text IS NULL OR old.blob_hash = $6
THEN COALESCE(to_timestamp($4), NOW()) ELSE f.updated_at END,
updated_by = CASE WHEN $6::text IS NULL OR old.blob_hash = $6
THEN $5 ELSE f.updated_by END
FROM old
WHERE f.id = old.id
RETURNING old.blob_hash, EXTRACT(EPOCH FROM f.updated_at)::bigint
RETURNING old.blob_hash, EXTRACT(EPOCH FROM f.updated_at)::bigint,
($6::text IS NULL OR old.blob_hash = $6)
"#,
)
.bind(new_hash)
.bind(new_size)
.bind(file_id)
.bind(modified_at.map(|t| t as f64))
.bind(caller_id)
.fetch_optional(self.pool.as_ref())
})
.await
{
Ok(Some(row)) => row,
Ok(None) => {
// File not found — compensate: remove the new blob ref
if let Err(e) = self.dedup.remove_reference(new_hash).await {
tracing::error!("Blob orphaned after missing file: {}", e);
)
.bind(new_hash)
.bind(new_size)
.bind(file_id)
.bind(modified_at.map(|t| t as f64))
.bind(caller_id)
.bind(expected_hash)
.fetch_optional(self.pool.as_ref())
})
.await
{
Ok(Some(row)) => row,
Ok(None) => {
// File not found — compensate: remove the new blob ref
if let Err(e) = self.dedup.remove_reference(new_hash).await {
tracing::error!("Blob orphaned after missing file: {}", e);
}
return Err(DomainError::not_found("File", file_id));
}
return Err(DomainError::not_found("File", file_id));
}
Err(e) => {
// UPDATE failed — compensate: remove the new blob ref
if let Err(rollback_err) = self.dedup.remove_reference(new_hash).await {
tracing::error!(
"Blob orphaned after failed UPDATE — hash: {}, err: {}",
&new_hash[..12],
rollback_err
);
Err(e) => {
// UPDATE failed — compensate: remove the new blob ref
if let Err(rollback_err) = self.dedup.remove_reference(new_hash).await {
tracing::error!(
"Blob orphaned after failed UPDATE — hash: {}, err: {}",
&new_hash[..12],
rollback_err
);
}
return Err(DomainError::internal_error(
"FileBlobWrite",
format!("update: {e}"),
));
}
return Err(DomainError::internal_error(
"FileBlobWrite",
format!("update: {e}"),
));
};
if !matched {
// CAS lost the race — some other writer's content is now the
// row's truth. Release the blob we ingested for nothing;
// nothing was written.
if let Err(e) = self.dedup.remove_reference(new_hash).await {
tracing::error!("Blob orphaned after CAS mismatch: {}", e);
}
};
return Err(DomainError::precondition_failed(
"File",
"content was modified concurrently",
));
}
// Decrement old blob ref (only if hash changed, best-effort)
if old_hash != new_hash
@@ -290,20 +307,22 @@ impl FileBlobWriteRepository {
// attempt's error falls through untouched so the 23505 mapping holds
// (a retried INSERT can legitimately lose to a concurrent identical
// upload).
// Post-D7: `user_id` omitted from the INSERT column list and the
// parent CTE. `drive_id` alone is the inherit-from-parent axis;
// provenance is `created_by` / `updated_by` (§14).
let result = retry_on_deadlock("files.insert", || {
sqlx::query_as::<_, (String, Uuid, String, i64, i64, Option<Uuid>, Option<Uuid>)>(
sqlx::query_as::<_, (String, String, i64, i64, Option<Uuid>, Option<Uuid>)>(
r#"
WITH parent AS (
SELECT id, user_id, drive_id, path FROM storage.folders WHERE id = $2::uuid
SELECT id, drive_id, path FROM storage.folders WHERE id = $2::uuid
)
INSERT INTO storage.files
(name, folder_id, user_id, drive_id, blob_hash, size,
(name, folder_id, drive_id, blob_hash, size,
mime_type, category_order, created_by, updated_by)
SELECT $1, parent.id, parent.user_id, parent.drive_id, $3, $4,
SELECT $1, parent.id, parent.drive_id, $3, $4,
$5, $6, $7, $7
FROM parent
RETURNING id::text,
user_id,
(SELECT path FROM parent),
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint,
@@ -322,71 +341,70 @@ impl FileBlobWriteRepository {
})
.await;
let (id, user_id, folder_path, created_at, updated_at, created_by, updated_by) =
match result {
Ok(Some(row)) => row,
Ok(None) => {
if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await {
tracing::error!(
"Blob orphaned after missing parent folder — hash: {}, err: {}",
&blob_hash[..12],
rollback_err
);
}
return Err(DomainError::not_found("Folder", fid));
let (id, folder_path, created_at, updated_at, created_by, updated_by) = match result {
Ok(Some(row)) => row,
Ok(None) => {
if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await {
tracing::error!(
"Blob orphaned after missing parent folder — hash: {}, err: {}",
&blob_hash[..12],
rollback_err
);
}
Err(e) => {
if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await {
tracing::error!(
"Blob orphaned after failed INSERT — hash: {}, err: {}",
&blob_hash[..12],
rollback_err
);
}
if let sqlx::Error::Database(ref db_err) = e
&& db_err.code().as_deref() == Some("23505")
{
// Idempotent re-upload: if the conflicting file already
// holds IDENTICAL content (same folder, same name, same
// blob hash), treat this as success and return that file
// instead of erroring. Re-uploading a partially-uploaded
// folder then becomes a clean no-op for everything that
// already landed — only the genuinely missing files
// transfer — instead of surfacing hundreds of spurious
// "already exists" failures. The duplicate blob reference
// taken during ingest was just released above, so the
// existing file's own reference is the only one (correct);
// a different-content clash still returns the conflict.
match self.fetch_identical_file(fid, &name, blob_hash).await {
Ok(Some(existing)) => {
tracing::info!(
"♻️ IDEMPOTENT UPLOAD: {} already present, identical content (hash: {})",
name,
&blob_hash[..12]
);
return Ok(existing);
}
Ok(None) => {} // genuine conflict (different content)
Err(lookup_err) => {
tracing::warn!(
"idempotency lookup failed for {} (hash {}): {} — returning conflict",
name,
&blob_hash[..12],
lookup_err
);
}
return Err(DomainError::not_found("Folder", fid));
}
Err(e) => {
if let Err(rollback_err) = self.dedup.remove_reference(blob_hash).await {
tracing::error!(
"Blob orphaned after failed INSERT — hash: {}, err: {}",
&blob_hash[..12],
rollback_err
);
}
if let sqlx::Error::Database(ref db_err) = e
&& db_err.code().as_deref() == Some("23505")
{
// Idempotent re-upload: if the conflicting file already
// holds IDENTICAL content (same folder, same name, same
// blob hash), treat this as success and return that file
// instead of erroring. Re-uploading a partially-uploaded
// folder then becomes a clean no-op for everything that
// already landed — only the genuinely missing files
// transfer — instead of surfacing hundreds of spurious
// "already exists" failures. The duplicate blob reference
// taken during ingest was just released above, so the
// existing file's own reference is the only one (correct);
// a different-content clash still returns the conflict.
match self.fetch_identical_file(fid, &name, blob_hash).await {
Ok(Some(existing)) => {
tracing::info!(
"♻️ IDEMPOTENT UPLOAD: {} already present, identical content (hash: {})",
name,
&blob_hash[..12]
);
return Ok(existing);
}
Ok(None) => {} // genuine conflict (different content)
Err(lookup_err) => {
tracing::warn!(
"idempotency lookup failed for {} (hash {}): {} — returning conflict",
name,
&blob_hash[..12],
lookup_err
);
}
return Err(DomainError::already_exists(
"File",
format!("'{name}' already exists in this folder"),
));
}
return Err(DomainError::internal_error(
"FileBlobWrite",
format!("insert: {e}"),
return Err(DomainError::already_exists(
"File",
format!("'{name}' already exists in this folder"),
));
}
};
return Err(DomainError::internal_error(
"FileBlobWrite",
format!("insert: {e}"),
));
}
};
tracing::info!(
"📡 STREAMING WRITE: {} ({} bytes, hash: {})",
@@ -404,7 +422,6 @@ impl FileBlobWriteRepository {
content_type,
created_at,
updated_at,
Some(user_id),
blob_hash.to_string(),
created_by,
updated_by,
@@ -421,11 +438,12 @@ impl FileBlobWriteRepository {
name: &str,
blob_hash: &str,
) -> Result<Option<File>, DomainError> {
// Post-D7: `f.user_id` is nullable on new rows; use
// `Option<Uuid>` to accept NULL.
let row = sqlx::query_as::<
_,
(
String,
Uuid,
String,
i64,
i64,
@@ -436,7 +454,7 @@ impl FileBlobWriteRepository {
),
>(
r#"
SELECT f.id::text, f.user_id, fo.path,
SELECT f.id::text, fo.path,
EXTRACT(EPOCH FROM f.created_at)::bigint,
EXTRACT(EPOCH FROM f.updated_at)::bigint,
f.created_by, f.updated_by, f.size, f.mime_type
@@ -460,7 +478,6 @@ impl FileBlobWriteRepository {
let Some((
id,
user_id,
folder_path,
created_at,
updated_at,
@@ -482,7 +499,6 @@ impl FileBlobWriteRepository {
mime_type,
created_at,
updated_at,
Some(user_id),
blob_hash.to_string(),
created_by,
updated_by,
@@ -535,11 +551,10 @@ impl FileWritePort for FileBlobWriteRepository {
>(
r#"
WITH dest AS (
SELECT user_id, drive_id FROM storage.folders WHERE id = $1::uuid
SELECT drive_id FROM storage.folders WHERE id = $1::uuid
)
UPDATE storage.files f
SET folder_id = $1::uuid,
user_id = COALESCE((SELECT user_id FROM dest), f.user_id),
drive_id = COALESCE((SELECT drive_id FROM dest), f.drive_id),
updated_at = NOW(),
updated_by = $3
@@ -568,7 +583,6 @@ impl FileWritePort for FileBlobWriteRepository {
row.4,
row.5,
row.6,
None,
String::new(),
row.7,
row.8,
@@ -611,29 +625,27 @@ impl FileWritePort for FileBlobWriteRepository {
>(
r#"
WITH src AS (
SELECT name, folder_id, user_id, blob_hash, size, mime_type, category_order
SELECT name, folder_id, blob_hash, size, mime_type, category_order
FROM storage.files
WHERE id = $1::uuid AND NOT is_trashed
),
-- The destination folder may differ from the source's
-- folder (when $2 is set); derive drive_id from the
-- DESTINATION so cross-drive copies land in the right
-- drive. Files in personal drives only copy within the
-- same drive today, but the join makes the migration
-- future-proof for D2's cross-drive copy story.
-- drive. Post-D7: `user_id` no longer projected — the
-- column is not written on new rows.
dest_folder AS (
SELECT id, user_id, drive_id
SELECT id, drive_id
FROM storage.folders
WHERE id = COALESCE($2::uuid,
(SELECT folder_id FROM src))
),
new_file AS (
INSERT INTO storage.files
(name, folder_id, user_id, drive_id, blob_hash, size,
(name, folder_id, drive_id, blob_hash, size,
mime_type, category_order, created_by, updated_by)
SELECT COALESCE($3::text, src.name),
dest_folder.id,
dest_folder.user_id,
dest_folder.drive_id,
src.blob_hash,
src.size,
@@ -642,14 +654,33 @@ impl FileWritePort for FileBlobWriteRepository {
$4,
$4
FROM src, dest_folder
RETURNING id::text, name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint,
RETURNING id,
id::text AS id_text,
name, folder_id::text, size, mime_type,
EXTRACT(EPOCH FROM created_at)::bigint AS created_at,
EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at,
blob_hash,
created_by,
updated_by
),
-- RFC 4918 §8.8 — dead properties MUST be duplicated on
-- COPY. With the id-keyed store (migration
-- 20260830000001) this is a single batch INSERT keyed on
-- the new file's id. Runs in the same query as the file
-- INSERT so either both land or neither does — atomic
-- by virtue of being one statement.
dead_prop_copy AS (
INSERT INTO storage.webdav_dead_properties
(file_id, namespace, local_name, value)
SELECT (SELECT id FROM new_file),
dp.namespace, dp.local_name, dp.value
FROM storage.webdav_dead_properties dp
WHERE dp.file_id = $1::uuid
)
SELECT * FROM new_file
SELECT id_text, name, folder_id, size, mime_type,
created_at, updated_at,
blob_hash, created_by, updated_by
FROM new_file
"#,
)
.bind(file_id)
@@ -699,7 +730,6 @@ impl FileWritePort for FileBlobWriteRepository {
row.4,
row.5,
row.6,
None,
row.7,
row.8,
row.9,
@@ -762,7 +792,6 @@ impl FileWritePort for FileBlobWriteRepository {
row.4,
row.5,
row.6,
None,
String::new(),
row.7,
row.8,
@@ -796,12 +825,20 @@ impl FileWritePort for FileBlobWriteRepository {
size: u64,
modified_at: Option<i64>,
caller_id: Uuid,
expected_hash: Option<&str>,
) -> Result<(String, i64), DomainError> {
// The content was already ingested into the chunk store by the
// upload-ingest layer; swap_blob_hash consumes its reference and
// releases it on failure.
let swapped = self
.swap_blob_hash(file_id, blob_hash, size as i64, modified_at, caller_id)
.swap_blob_hash(
file_id,
blob_hash,
size as i64,
modified_at,
caller_id,
expected_hash,
)
.await?;
// The file now maps to a different blob — drop the read-side cache
// entry so streaming downloads cannot serve the previous content
@@ -818,45 +855,84 @@ impl FileWritePort for FileBlobWriteRepository {
size: u64,
caller_id: Uuid,
) -> Result<(File, PathBuf), DomainError> {
let (user_id, drive_id) = self.resolve_owner_and_drive(folder_id.as_deref()).await?;
// For deferred registration we use a placeholder hash.
// The write-behind cache will call update_file_content later.
let placeholder_hash = "0000000000000000000000000000000000000000000000000000000000000000";
// §14: `created_by = $9 = updated_by = caller_id`. The legacy
// `user_id` column (dropped in D7) stays bound to the parent
// folder's owner; only the two provenance columns flip to the
// caller — see save_file_with_blob_impl.
let row = retry_on_deadlock("files.insert_deferred", || {
sqlx::query_as::<_, (String, i64, i64, Option<Uuid>, Option<Uuid>)>(
r#"
INSERT INTO storage.files
(name, folder_id, user_id, drive_id, blob_hash, size,
mime_type, category_order, created_by, updated_by)
VALUES ($1, $2::uuid, $3, $4, $5, $6, $7, $8, $9, $9)
RETURNING id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint,
created_by,
updated_by
"#,
)
.bind(&name)
.bind(&folder_id)
.bind(user_id)
.bind(drive_id)
.bind(placeholder_hash)
.bind(size as i64)
.bind(&content_type)
.bind(category_order_for(&name, &content_type))
.bind(caller_id)
.fetch_one(self.pool.as_ref())
})
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("deferred: {e}")))?;
// Post-D7: `user_id` omitted from the INSERT column list.
// §14: `created_by = <caller> = updated_by`.
//
// With a parent folder this is the SAME single-round-trip `WITH
// parent AS (…) INSERT … RETURNING` template `persist_file` uses:
// the old shape ran three queries per uploaded file — parent drive
// SELECT, INSERT, parent path SELECT — with the first and third
// re-reading the identical folders row (benches/ROUND11.md
// §Q1: 3 → 1 round-trips on the default REST upload path).
let (row, folder_path) = if let Some(fid) = folder_id.as_deref() {
let row = retry_on_deadlock("files.insert_deferred", || {
sqlx::query_as::<_, (String, String, i64, i64, Option<Uuid>, Option<Uuid>)>(
r#"
WITH parent AS (
SELECT id, drive_id, path FROM storage.folders WHERE id = $2::uuid
)
INSERT INTO storage.files
(name, folder_id, drive_id, blob_hash, size,
mime_type, category_order, created_by, updated_by)
SELECT $1, parent.id, parent.drive_id, $3, $4, $5, $6, $7, $7
FROM parent
RETURNING id::text,
(SELECT path FROM parent),
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint,
created_by,
updated_by
"#,
)
.bind(&name)
.bind(fid)
.bind(placeholder_hash)
.bind(size as i64)
.bind(&content_type)
.bind(category_order_for(&name, &content_type))
.bind(caller_id)
.fetch_optional(self.pool.as_ref())
})
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("deferred: {e}")))?
// 0 rows ⇒ the parent folder doesn't exist — same not-found the
// old `resolve_parent_drive` first query produced.
.ok_or_else(|| DomainError::not_found("Folder", fid))?;
((row.0, row.2, row.3, row.4, row.5), Some(row.1))
} else {
let drive_id = self.resolve_parent_drive(None).await?;
let row = retry_on_deadlock("files.insert_deferred", || {
sqlx::query_as::<_, (String, i64, i64, Option<Uuid>, Option<Uuid>)>(
r#"
INSERT INTO storage.files
(name, folder_id, drive_id, blob_hash, size,
mime_type, category_order, created_by, updated_by)
VALUES ($1, NULL, $2, $3, $4, $5, $6, $7, $7)
RETURNING id::text,
EXTRACT(EPOCH FROM created_at)::bigint,
EXTRACT(EPOCH FROM updated_at)::bigint,
created_by,
updated_by
"#,
)
.bind(&name)
.bind(drive_id)
.bind(placeholder_hash)
.bind(size as i64)
.bind(&content_type)
.bind(category_order_for(&name, &content_type))
.bind(caller_id)
.fetch_one(self.pool.as_ref())
})
.await
.map_err(|e| DomainError::internal_error("FileBlobWrite", format!("deferred: {e}")))?;
(row, None)
};
let folder_path = self.lookup_folder_path(folder_id.as_deref()).await?;
let file = Self::row_to_file(
row.0.clone(),
name,
@@ -866,7 +942,6 @@ impl FileWritePort for FileBlobWriteRepository {
content_type,
row.1,
row.2,
Some(user_id),
String::new(),
row.3,
row.4,
File diff suppressed because it is too large Load Diff
@@ -22,6 +22,21 @@ struct PlaylistRow {
updated_at: DateTime<Utc>,
}
/// A public playlist row carrying its aggregated track count, produced by the
/// single `LEFT JOIN … GROUP BY` that replaces the per-playlist `COUNT(*)` N+1.
#[derive(FromRow)]
struct PublicPlaylistCountRow {
id: Uuid,
name: String,
description: Option<String>,
owner_id: Uuid,
is_public: bool,
cover_file_id: Option<Uuid>,
created_at: DateTime<Utc>,
updated_at: DateTime<Utc>,
track_count: i64,
}
#[derive(FromRow)]
struct PlaylistItemRow {
id: Uuid,
@@ -79,6 +94,52 @@ impl PlaylistPgRepository {
pub fn pool(&self) -> &PgPool {
&self.pool
}
/// Public playlists together with their track counts in a **single**
/// round-trip. Replaces the adapter's 1 + N shape (one listing SELECT then
/// one `SELECT COUNT(*) FROM audio.playlist_items` per returned playlist —
/// up to 101 round-trips at `limit = 100`) with one `LEFT JOIN … GROUP BY`,
/// backed by `idx_playlist_items_playlist_id` (benches/ROUND25.md §Q1).
pub async fn list_public_playlists_with_counts(
&self,
limit: i64,
offset: i64,
) -> PlaylistRepositoryResult<Vec<(Playlist, i64)>> {
let rows = sqlx::query_as::<_, PublicPlaylistCountRow>(
"SELECT p.id, p.name, p.description, p.owner_id, p.is_public, p.cover_file_id, \
p.created_at, p.updated_at, COUNT(pi.id) AS track_count \
FROM audio.playlists p \
LEFT JOIN audio.playlist_items pi ON pi.playlist_id = p.id \
WHERE p.is_public = TRUE \
GROUP BY p.id \
ORDER BY p.updated_at DESC LIMIT $1 OFFSET $2",
)
.bind(limit)
.bind(offset)
.fetch_all(&*self.pool)
.await
.map_err(|e| {
DomainError::database_error(format!("Failed to list public playlists: {}", e))
})?;
rows.into_iter()
.map(|row| {
let track_count = row.track_count;
Playlist::with_id(
row.id,
row.name,
row.description,
row.owner_id,
row.is_public,
row.cover_file_id,
row.created_at,
row.updated_at,
)
.map(|p| (p, track_count))
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Playlist", e.to_string()))
})
.collect()
}
}
impl PlaylistRepository for PlaylistPgRepository {
@@ -180,6 +241,35 @@ impl PlaylistRepository for PlaylistPgRepository {
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Playlist", e.to_string()))
}
async fn find_playlists_by_ids(&self, ids: &[Uuid]) -> PlaylistRepositoryResult<Vec<Playlist>> {
if ids.is_empty() {
return Ok(Vec::new());
}
let rows = sqlx::query_as::<_, PlaylistRow>(
"SELECT id, name, description, owner_id, is_public, cover_file_id, created_at, updated_at FROM audio.playlists WHERE id = ANY($1)",
)
.bind(ids)
.fetch_all(&*self.pool)
.await
.map_err(|e| DomainError::database_error(format!("Failed to find playlists: {}", e)))?;
rows.into_iter()
.map(|row| {
Playlist::with_id(
row.id,
row.name,
row.description,
row.owner_id,
row.is_public,
row.cover_file_id,
row.created_at,
row.updated_at,
)
.map_err(|e| DomainError::new(ErrorKind::InternalError, "Playlist", e.to_string()))
})
.collect()
}
async fn list_playlists_by_owner(
&self,
owner_id: Uuid,
@@ -515,17 +605,27 @@ impl PlaylistItemRepository for PlaylistItemPgRepository {
playlist_id: &Uuid,
item_ids: &[Uuid],
) -> PlaylistItemRepositoryResult<()> {
for (index, item_id) in item_ids.iter().enumerate() {
sqlx::query(
"UPDATE audio.playlist_items SET position = $2 WHERE id = $1 AND playlist_id = $3",
)
.bind(item_id)
.bind(index as i32)
.bind(playlist_id)
.execute(&*self.pool)
.await
.map_err(|e| DomainError::database_error(format!("Failed to reorder: {}", e)))?;
if item_ids.is_empty() {
return Ok(());
}
// One UNNEST-driven UPDATE instead of one autocommit round-trip per
// track — a full drag-reorder of an N-track playlist was N statements
// (and non-atomic: a mid-loop failure left a half-applied order).
// `WITH ORDINALITY` numbers the ids in array order, 1-based, so
// `ord - 1` reproduces the historical 0-based positions.
sqlx::query(
r#"
UPDATE audio.playlist_items AS pi
SET position = (t.ord - 1)::int
FROM unnest($1::uuid[]) WITH ORDINALITY AS t(id, ord)
WHERE pi.id = t.id AND pi.playlist_id = $2
"#,
)
.bind(item_ids)
.bind(playlist_id)
.execute(&*self.pool)
.await
.map_err(|e| DomainError::database_error(format!("Failed to reorder: {}", e)))?;
Ok(())
}
@@ -21,18 +21,20 @@ impl RecentItemsPgRepository {
impl RecentItemsRepositoryPort for RecentItemsPgRepository {
async fn get_recent_items(&self, user_id: Uuid, limit: i32) -> Result<Vec<RecentItemDto>> {
// Binary UUID decode + app-side render (ROUND6 §10 pattern) — no
// server-side `::TEXT` casts, 16 B per id on the wire instead of 36.
let rows = sqlx::query(
r#"
SELECT
ur.id::TEXT AS "id",
ur.user_id::TEXT AS "user_id",
ur.id AS "id",
ur.user_id AS "user_id",
ur.item_id AS "item_id",
ur.item_type AS "item_type",
ur.accessed_at AS "accessed_at",
COALESCE(f.name, fld.name) AS "item_name",
f.size AS "item_size",
f.mime_type AS "item_mime_type",
COALESCE(f.folder_id::TEXT, fld.parent_id::TEXT) AS "parent_id",
COALESCE(f.folder_id, fld.parent_id) AS "parent_id",
CASE
WHEN ur.item_type = 'folder' THEN fld.path
WHEN ur.item_type = 'file' THEN COALESCE(pfld.path || '/' || f.name, f.name)
@@ -67,15 +69,19 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
.iter()
.map(|row| {
RecentItemDto {
id: row.get("id"),
user_id: row.get("user_id"),
id: row.get::<i32, _>("id").to_string(),
user_id: row.get::<Uuid, _>("user_id").to_string(),
item_id: row.get("item_id"),
item_type: row.get("item_type"),
accessed_at: row.get("accessed_at"),
item_name: row.try_get("item_name").ok(),
item_size: row.try_get("item_size").ok(),
item_mime_type: row.try_get("item_mime_type").ok(),
parent_id: row.try_get("parent_id").ok(),
parent_id: row
.try_get::<Option<Uuid>, _>("parent_id")
.ok()
.flatten()
.map(|u| u.to_string()),
item_path: row.try_get("item_path").ok(),
// Temporary defaults; with_display_fields() computes the real values
icon_class: String::new(),
@@ -90,19 +96,24 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
Ok(items)
}
async fn upsert_access(&self, user_id: Uuid, item_id: &str, item_type: &str) -> Result<()> {
sqlx::query(
async fn upsert_access(&self, user_id: Uuid, item_id: &str, item_type: &str) -> Result<bool> {
// `xmax = 0` on the affected row is the canonical upsert idiom for
// "this was an INSERT, not a DO UPDATE" — lets the caller skip the
// prune round-trip on the common re-access (UPDATE) path
// (benches/ROUND13.md §Q3).
let inserted: bool = sqlx::query_scalar(
r#"
INSERT INTO auth.user_recent_files (user_id, item_id, item_type, accessed_at)
VALUES ($1, $2, $3, CURRENT_TIMESTAMP)
ON CONFLICT (user_id, item_id, item_type)
DO UPDATE SET accessed_at = CURRENT_TIMESTAMP
RETURNING (xmax = 0)
"#,
)
.bind(user_id)
.bind(item_id)
.bind(item_type)
.execute(&*self.db_pool)
.fetch_one(&*self.db_pool)
.await
.map_err(|e| {
error!("Database error upserting recent item access: {}", e);
@@ -113,7 +124,7 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
)
})?;
Ok(())
Ok(inserted)
}
async fn remove_item(&self, user_id: Uuid, item_id: &str, item_type: &str) -> Result<bool> {
@@ -206,6 +217,20 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
// ── Build the UNION ALL CTE ─────────────────────────────────────────
let mut cte_branches: Vec<&str> = Vec::new();
// Post-D7: `is_owner` means "the caller holds an Owner
// role_grant on the drive owning this row". Personal drives:
// the single-owner invariant makes this trivially true for the
// owner and false for anyone else. Shared drives: multiple
// Owners possible; each of them gets `true`. Used only to gate
// whether the handler exposes the full path (path-hierarchy
// hiding for share recipients — see `recent_handler.rs`).
//
// The `created_by` projection is separate — §14 provenance,
// used for the "Owner" column and the owner sort's username
// JOIN. The two signals genuinely differ post-D2: e.g. Bob
// (Editor on Alice's shared drive) making a file has
// `created_by = Bob` but `is_owner = false` because Alice owns
// the drive.
let folder_branch = r#"
SELECT
'folder'::text AS resource_type,
@@ -216,9 +241,19 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
-1::bigint AS size,
fld.created_at AS resource_created_at,
fld.updated_at AS modified_at,
fld.user_id AS owner_id,
fld.drive_id AS drive_id,
NULL::text AS blob_hash,
(fld.user_id = $1::uuid) AS is_owner,
fld.created_by AS created_by,
fld.updated_by AS updated_by,
EXISTS (
SELECT 1 FROM storage.role_grants g
WHERE g.resource_type = 'drive'
AND g.resource_id = fld.drive_id
AND g.role = 'owner'
AND g.subject_type = 'user'
AND g.subject_id = $1::uuid
AND (g.expires_at IS NULL OR g.expires_at > NOW())
) AS is_owner,
ur.accessed_at AS accessed_at,
fld.path::text AS resource_path,
LOWER(fld.name) AS sort_str,
@@ -239,9 +274,19 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
f.size::bigint,
f.created_at AS resource_created_at,
f.updated_at AS modified_at,
f.user_id AS owner_id,
f.drive_id AS drive_id,
f.blob_hash,
(f.user_id = $1::uuid) AS is_owner,
f.created_by AS created_by,
f.updated_by AS updated_by,
EXISTS (
SELECT 1 FROM storage.role_grants g
WHERE g.resource_type = 'drive'
AND g.resource_id = f.drive_id
AND g.role = 'owner'
AND g.subject_type = 'user'
AND g.subject_id = $1::uuid
AND (g.expires_at IS NULL OR g.expires_at > NOW())
) AS is_owner,
ur.accessed_at AS accessed_at,
COALESCE(pfld.path::text || '/' || f.name, f.name) AS resource_path,
LOWER(f.name) AS sort_str,
@@ -392,7 +437,9 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
};
let user_join = if need_user_join {
"LEFT JOIN auth.users u ON u.id = r.owner_id"
// Post-D7: `owner_id` column retired; use `created_by`
// (§14 provenance) as the "owner" identity for the sort.
"LEFT JOIN auth.users u ON u.id = r.created_by"
} else {
""
};
@@ -409,7 +456,8 @@ impl RecentItemsRepositoryPort for RecentItemsPgRepository {
SELECT
r.resource_type, r.resource_id, r.name, r.parent_id,
r.mime_type, r.size, r.resource_created_at, r.modified_at,
r.owner_id, r.is_owner, r.accessed_at, r.resource_path,
r.drive_id, r.blob_hash, r.created_by, r.updated_by,
r.is_owner, r.accessed_at, r.resource_path,
r.sort_str, r.type_order, r.folder_first{username_col}
FROM resources r
{user_join}
@@ -484,8 +532,10 @@ LIMIT $6"
size,
resource_created_at: row.get("resource_created_at"),
modified_at: row.get("modified_at"),
owner_id: row.get("owner_id"),
drive_id: row.get("drive_id"),
blob_hash: row.try_get("blob_hash").ok(),
created_by: row.try_get("created_by").ok(),
updated_by: row.try_get("updated_by").ok(),
is_owner: row.try_get("is_owner").unwrap_or(false),
accessed_at: row.get("accessed_at"),
path: row.try_get("resource_path").ok(),
@@ -327,6 +327,77 @@ impl SessionStoragePort for SessionPgRepository {
.map_err(DomainError::from)
}
/// Revoke + insert + last-login stamp in ONE transaction — the refresh
/// rotation used to pay two full BEGIN/COMMIT round-trip pairs
/// (`revoke_session` then `create_session`) per token refresh.
async fn rotate_session(
&self,
old_session_id: Uuid,
new_session: Session,
) -> Result<Session, DomainError> {
let session_clone = new_session.clone();
with_transaction(&self.pool, "rotate_session", |tx| {
Box::pin(async move {
sqlx::query("UPDATE auth.sessions SET revoked = true WHERE id = $1")
.bind(old_session_id)
.execute(&mut **tx)
.await
.map_err(Self::map_sqlx_error)?;
sqlx::query(
r#"
INSERT INTO auth.sessions (
id, user_id, refresh_token, expires_at,
ip_address, user_agent, created_at, revoked, family_id
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9
)
"#,
)
.bind(session_clone.id())
.bind(session_clone.user_id())
.bind(session_clone.refresh_token())
.bind(session_clone.expires_at())
.bind(session_clone.ip_address())
.bind(session_clone.user_agent())
.bind(session_clone.created_at())
.bind(session_clone.is_revoked())
.bind(session_clone.family_id())
.execute(&mut **tx)
.await
.map_err(Self::map_sqlx_error)?;
sqlx::query(
r#"
UPDATE auth.users
SET last_login_at = NOW(), updated_at = NOW()
WHERE id = $1
"#,
)
.bind(session_clone.user_id())
.execute(&mut **tx)
.await
.map_err(|e| {
tracing::warn!(
"Could not update last_login_at for user {}: {}",
session_clone.user_id(),
e
);
SessionRepositoryError::DatabaseError(format!(
"Session rotated but could not update last_login_at: {}",
e
))
})?;
Ok(session_clone)
}) as BoxFuture<'_, SessionRepositoryResult<Session>>
})
.await
.map_err(DomainError::from)?;
Ok(new_session)
}
async fn get_session_by_refresh_token(
&self,
refresh_token: &str,
@@ -122,6 +122,35 @@ impl ShareStoragePort for SharePgRepository {
Self::row_to_entity(&row)
}
async fn increment_access_count(&self, token: &str) -> Result<u64, DomainError> {
// One atomic statement — the relative bump can't lose concurrent
// increments and never rewrites unrelated columns (the legacy
// read-modify-write wrote back item_name/password_hash wholesale,
// silently clobbering concurrent owner edits). The expiry guard
// mirrors find_share_by_token's MIN(expires_at) subquery: NULL =
// never expires.
let result = sqlx::query(
r#"
UPDATE storage.shares s
SET access_count = s.access_count + 1
WHERE s.token = $1
AND COALESCE(
(SELECT MIN(ag.expires_at)
FROM storage.role_grants ag
WHERE ag.subject_type = 'token' AND ag.subject_id = s.id) > NOW(),
TRUE)
"#,
)
.bind(token)
.execute(&*self.db_pool)
.await
.map_err(|e| {
tracing::error!("Database error incrementing share access count: {}", e);
DomainError::internal_error("Share", format!("Failed to register access: {e}"))
})?;
Ok(result.rows_affected())
}
async fn find_share_by_token(&self, token: &str) -> Result<Share, DomainError> {
let row = sqlx::query(
r#"
@@ -123,26 +123,45 @@ impl TrashRepository for TrashDbRepository {
}
async fn get_trash_items(&self, user_id: &Uuid) -> Result<Vec<TrashedItem>> {
let rows =
sqlx::query_as::<_, (Uuid, String, String, Uuid, Option<DateTime<Utc>>, String)>(
r#"
SELECT t.id, t.name, t.item_type, t.user_id, t.trashed_at,
// Post-D7: the `WHERE t.user_id = $1` filter is gone — the
// `user_id` column was dropped from `storage.{files,folders}`
// and the view no longer projects it. Scope is drive-membership
// via role_grants; group memberships expand inline through
// `storage.caller_group_ids`. Same predicate shape as
// `list_root_folders_for_caller` / the file listings.
//
// Legacy method — the paginated `list_resources_paged` is the
// modern shape and takes explicit drive_ids from the service
// layer.
let rows = sqlx::query_as::<_, (Uuid, String, String, Option<DateTime<Utc>>, String)>(
r#"
SELECT t.id, t.name, t.item_type, t.trashed_at,
COALESCE(p.path || '/' || t.name, t.name) AS original_path
FROM storage.trash_items t
LEFT JOIN storage.folders p ON p.id = t.original_parent_id
WHERE t.user_id = $1
WHERE EXISTS (
SELECT 1 FROM storage.role_grants g
WHERE g.resource_type = 'drive'
AND g.resource_id = t.drive_id
AND (g.expires_at IS NULL OR g.expires_at > NOW())
AND (
(g.subject_type = 'user' AND g.subject_id = $1)
OR (g.subject_type = 'group' AND g.subject_id IN
(SELECT storage.caller_group_ids($1)))
)
)
ORDER BY t.trashed_at DESC
"#,
)
.bind(user_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("TrashDb", format!("list: {e}")))?;
)
.bind(user_id)
.fetch_all(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("TrashDb", format!("list: {e}")))?;
Ok(rows
.into_iter()
.map(|(id, name, item_type, uid, trashed_at, path)| {
self.row_to_trashed_item(id, name, item_type, uid, trashed_at, path)
.map(|(id, name, item_type, trashed_at, path)| {
self.row_to_trashed_item(id, name, item_type, *user_id, trashed_at, path)
})
.collect())
}
@@ -153,9 +172,16 @@ impl TrashRepository for TrashDbRepository {
// …)` in the service callers (`restore_item`, `delete_permanently`).
// The drive precheck in `pg_acl_engine` then resolves Owner-on-drive
// → Delete-permission for items in shared drives.
let row = sqlx::query_as::<_, (Uuid, String, String, Uuid, Option<DateTime<Utc>>, String)>(
//
// Post-D7: `t.user_id` no longer exists — the column is dropped
// from `storage.{files,folders}` and no longer projected by the
// view. The entity's `user_id` field is still non-optional;
// synthesize `Uuid::nil()`. AuthZ decisions don't consult this
// field — they've already resolved the caller's role on the
// target's drive.
let row = sqlx::query_as::<_, (Uuid, String, String, Option<DateTime<Utc>>, String)>(
r#"
SELECT t.id, t.name, t.item_type, t.user_id, t.trashed_at,
SELECT t.id, t.name, t.item_type, t.trashed_at,
COALESCE(p.path || '/' || t.name, t.name) AS original_path
FROM storage.trash_items t
LEFT JOIN storage.folders p ON p.id = t.original_parent_id
@@ -167,8 +193,8 @@ impl TrashRepository for TrashDbRepository {
.await
.map_err(|e| DomainError::internal_error("TrashDb", format!("get: {e}")))?;
Ok(row.map(|(id, name, item_type, uid, trashed_at, path)| {
self.row_to_trashed_item(id, name, item_type, uid, trashed_at, path)
Ok(row.map(|(id, name, item_type, trashed_at, path)| {
self.row_to_trashed_item(id, name, item_type, Uuid::nil(), trashed_at, path)
}))
}
@@ -226,15 +252,35 @@ impl TrashRepository for TrashDbRepository {
async fn delete_expired_bulk(&self) -> Result<(u64, u64)> {
let cutoff = Utc::now() - chrono::Duration::days(self.retention_days);
// The `read_only` policy on a drive is a compliance-grade freeze:
// NO state on the drive changes while the policy is on, including
// background retention. The `JOIN storage.drives d ... AND
// (d.policies->>'read_only')::boolean IS NOT TRUE` filter excludes
// frozen drives at SELECT time. Retention clock keeps ticking; on
// unfreeze, the next sweep tick catches up on anything past its
// TTL. Legal-hold guarantee documented in `docs/plan/drive.md` §8
// and `docs/guide/trash.md`.
//
// `(policies->>'read_only')::boolean IS NOT TRUE` semantics:
// - key missing → NULL::boolean → IS NOT TRUE → included
// - explicit `false` → FALSE → IS NOT TRUE → included
// - explicit `true` → TRUE → IS TRUE → excluded
// Correct for both current data (most drives omit the key) and
// freshly-frozen drives.
// 1. Bulk-delete expired trashed files in batches.
// The PG trigger `trg_files_decrement_blob_ref` automatically
// decrements blob ref_count for every deleted row.
let files_deleted = self
.delete_expired_batch_loop(
"DELETE FROM storage.files
WHERE id IN (SELECT id FROM storage.files
WHERE is_trashed = TRUE AND trashed_at < $1
ORDER BY trashed_at
WHERE id IN (SELECT f.id
FROM storage.files f
JOIN storage.drives d ON d.id = f.drive_id
WHERE f.is_trashed = TRUE
AND f.trashed_at < $1
AND (d.policies->>'read_only')::boolean IS NOT TRUE
ORDER BY f.trashed_at
LIMIT $2)",
cutoff,
1_000,
@@ -244,13 +290,19 @@ impl TrashRepository for TrashDbRepository {
// 2. Bulk-delete expired trashed folders in batches.
// FK ON DELETE CASCADE handles descendant folders and their
// files, so each row can fan out to an entire subtree — hence
// the smaller batch size.
// the smaller batch size. Same read_only exclusion applies:
// a subtree rooted in a frozen drive isn't purged even if the
// folder's own trashed_at is past retention.
let folders_deleted = self
.delete_expired_batch_loop(
"DELETE FROM storage.folders
WHERE id IN (SELECT id FROM storage.folders
WHERE is_trashed = TRUE AND trashed_at < $1
ORDER BY trashed_at
WHERE id IN (SELECT f.id
FROM storage.folders f
JOIN storage.drives d ON d.id = f.drive_id
WHERE f.is_trashed = TRUE
AND f.trashed_at < $1
AND (d.policies->>'read_only')::boolean IS NOT TRUE
ORDER BY f.trashed_at
LIMIT $2)",
cutoff,
100,
@@ -313,9 +365,10 @@ impl TrashDbRepository {
-1::bigint AS size,
fld.created_at AS resource_created_at,
fld.updated_at AS modified_at,
fld.user_id AS owner_id,
fld.drive_id AS drive_id,
NULL::text AS blob_hash,
fld.created_by AS created_by,
fld.updated_by AS updated_by,
fld.trashed_at AS trashed_at,
(fld.trashed_at + ($7::int * INTERVAL '1 day')) AS deletion_date,
fld.path::text AS resource_path,
@@ -340,9 +393,10 @@ impl TrashDbRepository {
f.size::bigint AS size,
f.created_at AS resource_created_at,
f.updated_at AS modified_at,
f.user_id AS owner_id,
f.drive_id AS drive_id,
f.blob_hash,
f.created_by AS created_by,
f.updated_by AS updated_by,
f.trashed_at AS trashed_at,
(f.trashed_at + ($7::int * INTERVAL '1 day')) AS deletion_date,
COALESCE(pfld.path::text || '/' || f.name, f.name) AS resource_path,
@@ -472,7 +526,8 @@ impl TrashDbRepository {
SELECT
r.resource_type, r.resource_id, r.name, r.parent_id,
r.mime_type, r.size, r.resource_created_at, r.modified_at,
r.owner_id, r.drive_id, r.trashed_at, r.deletion_date, r.resource_path,
r.drive_id, r.blob_hash, r.created_by, r.updated_by,
r.trashed_at, r.deletion_date, r.resource_path,
r.sort_str, r.type_order, r.folder_first
FROM resources r
{keyset}
@@ -529,9 +584,10 @@ LIMIT $6"
size,
resource_created_at: row.get("resource_created_at"),
modified_at: row.get("modified_at"),
owner_id: row.get("owner_id"),
drive_id: row.get("drive_id"),
blob_hash: row.try_get("blob_hash").ok(),
created_by: row.try_get("created_by").ok(),
updated_by: row.try_get("updated_by").ok(),
trashed_at,
deletion_date,
path: row.try_get("resource_path").ok(),
@@ -85,6 +85,33 @@ impl UserPgRepository {
})
}
/// Fetch only `(storage_used_bytes, storage_quota_bytes)`. Not part of
/// the `UserRepository` trait — called from `StorageUsageService`.
///
/// Same rationale as [`Self::get_user_flags`]: the full-row SELECT drags
/// `image` (a data URI of up to 512 KiB), `password_hash`,
/// `ui_preferences`, … across the wire, and the quota path runs on every
/// folder PROPFIND and every upload quota check just to read two i64s.
/// Measured in `benches/QUOTA-PATH.md`.
pub async fn get_storage_usage(&self, id: Uuid) -> UserRepositoryResult<(i64, i64)> {
let row = sqlx::query(
r#"
SELECT storage_used_bytes, storage_quota_bytes
FROM auth.users
WHERE id = $1
"#,
)
.bind(id)
.fetch_one(&*self.pool)
.await
.map_err(Self::map_sqlx_error)?;
Ok((
row.get("storage_used_bytes"),
row.get("storage_quota_bytes"),
))
}
/// Updates a user's profile image (URL or data URI). Not part of the
/// `UserRepository` trait — called directly from `AuthApplicationService`.
pub async fn update_image(
@@ -106,6 +133,48 @@ impl UserPgRepository {
.map_err(Self::map_sqlx_error)?;
Ok(())
}
/// Shallow-merge a partial UI-preferences patch into
/// `ui_preferences`. The Postgres `||` operator merges top-level
/// keys — `{"a":1,"b":2} || {"b":3,"c":4}` → `{"a":1,"b":3,"c":4}`,
/// which is exactly the semantic PATCH callers want: a partial
/// write only touches the keys it mentions, so a preference set on
/// one device isn't wiped by a partial write from another.
///
/// `jsonb_strip_nulls` removes any key whose incoming value is
/// null, giving callers a documented delete-a-key path (`PATCH
/// {"foo": null}` clears `foo`). Nested nulls inside a value
/// object survive — we only strip at the top level via the merge
/// result.
///
/// Not part of the `UserRepository` trait — called directly from
/// `AuthApplicationService::update_profile`. Bumps `updated_at`
/// so the standard "when did this row change" audits stay useful.
///
/// The CHECK constraints
/// (`users_ui_preferences_is_object` + `_size_cap`) enforce shape
/// and cap at the schema layer; a violating patch surfaces as an
/// sqlx error and returns to the handler as 400.
pub async fn update_ui_preferences(
&self,
user_id: Uuid,
patch: &serde_json::Value,
) -> UserRepositoryResult<()> {
sqlx::query(
r#"
UPDATE auth.users
SET ui_preferences = jsonb_strip_nulls(ui_preferences || $2::jsonb),
updated_at = NOW()
WHERE id = $1
"#,
)
.bind(user_id)
.bind(patch)
.execute(&*self.pool)
.await
.map_err(Self::map_sqlx_error)?;
Ok(())
}
}
impl UserRepository for UserPgRepository {
@@ -123,18 +192,25 @@ impl UserRepository for UserPgRepository {
let role_str = user_clone.role().to_string();
// Modify the SQL to do an explicit cast to the auth.userrole type
// `image` is included here (was missing pre-fix); without
// it a JIT-provisioned OIDC user landed in the row with
// a NULL profile picture even when the IdP's `picture`
// claim was non-empty. `update_user` already wrote the
// column so existing-user re-logins worked, but the
// first-time INSERT silently dropped it — surfaced by
// tests/oidc/oidc.hurl Step 6 asserting on `$.image`.
let _result = sqlx::query(
r#"
INSERT INTO auth.users (
id, username, email, password_hash, role,
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, is_external,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at,
preferred_locale, notify_on_share
preferred_locale, notify_on_share, ui_preferences
) VALUES (
$1, $2, $3, $4, $5::auth.userrole, $6, $7, $8, $9, $10, $11,
$12, $13, $14, $15, $16, $17, $18, $19
$12, $13, $14, $15, $16, $17, $18, $19, $20, $21
)
RETURNING *
"#,
@@ -152,12 +228,17 @@ impl UserRepository for UserPgRepository {
.bind(user_clone.is_active())
.bind(user_clone.oidc_provider())
.bind(user_clone.oidc_subject())
.bind(user_clone.image())
.bind(user_clone.is_external())
.bind(user_clone.given_name())
.bind(user_clone.family_name())
.bind(user_clone.email_verified_at())
.bind(user_clone.preferred_locale())
.bind(user_clone.notify_on_share())
// ui_preferences bind: always a JSON object. `User::new`
// initialises the bag to `{}`; ownership stays with the
// repo for shallow-merge writes via `update_ui_preferences`.
.bind(user_clone.ui_preferences())
.execute(&mut **tx)
.await
.map_err(Self::map_sqlx_error)?;
@@ -182,7 +263,8 @@ impl UserRepository for UserPgRepository {
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
ui_preferences
FROM auth.users
WHERE id = $1
"#,
@@ -220,6 +302,7 @@ impl UserRepository for UserPgRepository {
row.get("email_verified_at"),
row.get("preferred_locale"),
row.get("notify_on_share"),
row.get::<serde_json::Value, _>("ui_preferences"),
))
}
@@ -232,7 +315,8 @@ impl UserRepository for UserPgRepository {
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
ui_preferences
FROM auth.users
WHERE username = $1
"#,
@@ -270,6 +354,7 @@ impl UserRepository for UserPgRepository {
row.get("email_verified_at"),
row.get("preferred_locale"),
row.get("notify_on_share"),
row.get::<serde_json::Value, _>("ui_preferences"),
))
}
@@ -282,7 +367,8 @@ impl UserRepository for UserPgRepository {
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
ui_preferences
FROM auth.users
WHERE email = $1
"#,
@@ -320,6 +406,7 @@ impl UserRepository for UserPgRepository {
row.get("email_verified_at"),
row.get("preferred_locale"),
row.get("notify_on_share"),
row.get::<serde_json::Value, _>("ui_preferences"),
))
}
@@ -327,6 +414,16 @@ impl UserRepository for UserPgRepository {
/// recipient expansion). Missing ids are silently skipped — the
/// caller treats absent rows as "no such recipient", same as
/// `get_user_by_id` returning `NotFound` for a single lookup.
///
/// Notification-recipient projection: the up-to-512 KiB avatar `image`
/// and the `ui_preferences` JSONB are NOT hydrated (both come back as
/// `None`/`Null`) — the sole caller
/// (`RecipientNotificationService`) reads only the email/eligibility
/// fields, and a group fan-out of M members otherwise detoasted +
/// shipped + parsed M avatars purely to discard them (the ROUND12 §Q1
/// avatar-narrowing pattern; benches/ROUND13.md §Q1). If a future
/// caller needs the avatar, add a wide sibling rather than widening
/// this one back.
async fn get_users_by_ids(&self, ids: Vec<Uuid>) -> UserRepositoryResult<Vec<User>> {
if ids.is_empty() {
return Ok(Vec::new());
@@ -338,7 +435,7 @@ impl UserRepository for UserPgRepository {
id, username, email, password_hash, role::text as role_text,
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
oidc_provider, oidc_subject, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share
FROM auth.users
WHERE id = ANY($1)
@@ -372,13 +469,14 @@ impl UserRepository for UserPgRepository {
row.get("active"),
row.get("oidc_provider"),
row.get("oidc_subject"),
row.get("image"),
None, // image — not projected (notification-recipient path)
row.get("is_external"),
row.get("given_name"),
row.get("family_name"),
row.get("email_verified_at"),
row.get("preferred_locale"),
row.get("notify_on_share"),
serde_json::Value::Null, // ui_preferences — not projected
)
})
.collect())
@@ -410,7 +508,19 @@ impl UserRepository for UserPgRepository {
family_name = $13,
email_verified_at = $14,
preferred_locale = $15,
notify_on_share = $16
notify_on_share = $16,
-- Include `is_external` so the external →
-- internal upgrade path
-- (`AuthApplicationService::upgrade_to_internal`)
-- can flip this flag. Previously omitted
-- because no code path mutated it after
-- creation. The DB CHECK
-- `users_external_no_storage`
-- (`is_external=false OR quota=0`) is
-- satisfied by the upgrade because it
-- writes both fields in the same UPDATE:
-- `is_external=false, quota>0`.
is_external = $17
WHERE id = $1
"#,
)
@@ -430,6 +540,7 @@ impl UserRepository for UserPgRepository {
.bind(user_clone.email_verified_at())
.bind(user_clone.preferred_locale())
.bind(user_clone.notify_on_share())
.bind(user_clone.is_external())
.execute(&mut **tx)
.await
.map_err(Self::map_sqlx_error)?;
@@ -506,7 +617,8 @@ impl UserRepository for UserPgRepository {
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
ui_preferences
FROM auth.users
WHERE ($3 OR is_external = FALSE)
ORDER BY created_at DESC
@@ -551,6 +663,7 @@ impl UserRepository for UserPgRepository {
row.get("email_verified_at"),
row.get("preferred_locale"),
row.get("notify_on_share"),
row.get::<serde_json::Value, _>("ui_preferences"),
)
})
.collect();
@@ -572,7 +685,8 @@ impl UserRepository for UserPgRepository {
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
ui_preferences
FROM auth.users
WHERE (username ILIKE $1 OR email ILIKE $1)
AND ($3 OR is_external = FALSE)
@@ -617,6 +731,7 @@ impl UserRepository for UserPgRepository {
row.get("email_verified_at"),
row.get("preferred_locale"),
row.get("notify_on_share"),
row.get::<serde_json::Value, _>("ui_preferences"),
)
})
.collect();
@@ -695,6 +810,15 @@ impl UserRepository for UserPgRepository {
Ok(())
}
/// Counts users by role with a scalar `COUNT(*)` — no row hydration.
async fn count_users_by_role(&self, role: &str) -> UserRepositoryResult<i64> {
sqlx::query_scalar("SELECT COUNT(*) FROM auth.users WHERE role::text = $1")
.bind(role)
.fetch_one(&*self.pool)
.await
.map_err(Self::map_sqlx_error)
}
/// Lists users by role
async fn list_users_by_role(&self, role: &str) -> UserRepositoryResult<Vec<User>> {
let rows = sqlx::query(
@@ -704,7 +828,8 @@ impl UserRepository for UserPgRepository {
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
ui_preferences
FROM auth.users
WHERE role::text = $1
ORDER BY created_at DESC
@@ -746,6 +871,7 @@ impl UserRepository for UserPgRepository {
row.get("email_verified_at"),
row.get("preferred_locale"),
row.get("notify_on_share"),
row.get::<serde_json::Value, _>("ui_preferences"),
)
})
.collect();
@@ -782,7 +908,8 @@ impl UserRepository for UserPgRepository {
storage_quota_bytes, storage_used_bytes,
created_at, updated_at, last_login_at, active,
oidc_provider, oidc_subject, image, is_external,
given_name, family_name, email_verified_at, preferred_locale, notify_on_share
given_name, family_name, email_verified_at, preferred_locale, notify_on_share,
ui_preferences
FROM auth.users
WHERE oidc_provider = $1 AND oidc_subject = $2
"#,
@@ -820,6 +947,7 @@ impl UserRepository for UserPgRepository {
row.get("email_verified_at"),
row.get("preferred_locale"),
row.get("notify_on_share"),
row.get::<serde_json::Value, _>("ui_preferences"),
))
}
@@ -957,12 +1085,93 @@ impl UserStoragePort for UserPgRepository {
.map_err(DomainError::from)
}
async fn search_usernames(
&self,
query: &str,
limit: i64,
include_external: bool,
) -> Result<Vec<Option<String>>, DomainError> {
// Same predicate / order / limit as `search_users`, username-only
// projection — the sharee autocomplete path reads nothing else, and
// the wide row drags the avatar `image` per matched user.
let pattern = format!("%{}%", query);
let rows = sqlx::query(
r#"
SELECT username
FROM auth.users
WHERE (username ILIKE $1 OR email ILIKE $1)
AND ($3 OR is_external = FALSE)
ORDER BY username
LIMIT $2
"#,
)
.bind(&pattern)
.bind(limit)
.bind(include_external)
.fetch_all(&*self.pool)
.await
.map_err(Self::map_sqlx_error)
.map_err(DomainError::from)?;
Ok(rows.into_iter().map(|row| row.get("username")).collect())
}
async fn mark_email_verified(&self, user_id: Uuid) -> Result<(), DomainError> {
// SQL twin of `User::mark_email_verified` — stamps once, keeps the
// first timestamp, and touches only the two columns involved.
sqlx::query(
r#"
UPDATE auth.users
SET email_verified_at = NOW(), updated_at = NOW()
WHERE id = $1 AND email_verified_at IS NULL
"#,
)
.bind(user_id)
.execute(&*self.pool)
.await
.map_err(Self::map_sqlx_error)
.map_err(DomainError::from)?;
Ok(())
}
async fn sync_oidc_login_profile(
&self,
user_id: Uuid,
image: Option<&str>,
) -> Result<(), DomainError> {
// `IS DISTINCT FROM` guard (the `update_storage_usage` pattern): the
// common repeat-login case — same IdP avatar, already verified —
// writes nothing at all (no dead tuple, no WAL).
sqlx::query(
r#"
UPDATE auth.users
SET image = $2,
email_verified_at = COALESCE(email_verified_at, NOW()),
updated_at = NOW()
WHERE id = $1
AND (image IS DISTINCT FROM $2 OR email_verified_at IS NULL)
"#,
)
.bind(user_id)
.bind(image)
.execute(&*self.pool)
.await
.map_err(Self::map_sqlx_error)
.map_err(DomainError::from)?;
Ok(())
}
async fn list_users_by_role(&self, role: &str) -> Result<Vec<User>, DomainError> {
UserRepository::list_users_by_role(self, role)
.await
.map_err(DomainError::from)
}
async fn count_users_by_role(&self, role: &str) -> Result<i64, DomainError> {
UserRepository::count_users_by_role(self, role)
.await
.map_err(DomainError::from)
}
async fn delete_user(&self, user_id: Uuid) -> Result<(), DomainError> {
UserRepository::delete_user(self, user_id)
.await
+105 -43
View File
@@ -9,7 +9,7 @@ use std::pin::Pin;
use azure_storage::StorageCredentials;
use azure_storage_blobs::prelude::*;
use bytes::Bytes;
use futures::StreamExt;
use futures::{StreamExt, TryStreamExt};
use tokio::fs;
use crate::application::ports::blob_storage_ports::{
@@ -33,8 +33,21 @@ impl AzureBlobBackend {
StorageCredentials::access_key(&config.account_name, config.account_key.clone())
};
let container_client = ClientBuilder::new(&config.account_name, credentials)
.container_client(&config.container);
// Custom endpoint (Azurite emulator / private deployment /
// benches) mirrors S3's `endpoint_url`; default is the public
// cloud URL derived from the account name.
let container_client = match &config.endpoint_url {
Some(uri) => ClientBuilder::with_location(
azure_storage::CloudLocation::Custom {
account: config.account_name.clone(),
uri: uri.trim_end_matches('/').to_string(),
},
credentials,
)
.container_client(&config.container),
None => ClientBuilder::new(&config.account_name, credentials)
.container_client(&config.container),
};
Self {
container_client,
@@ -130,7 +143,9 @@ impl BlobStorageBackend for AzureBlobBackend {
return Ok(size);
}
client.put_block_blob(data.to_vec()).await.map_err(|e| {
// `Bytes` converts into `azure_core::Body` by reference count —
// the old `data.to_vec()` copied every chunk once more.
client.put_block_blob(data).await.map_err(|e| {
DomainError::internal_error("Azure", format!("Failed to upload blob {hash}: {e}"))
})?;
@@ -138,6 +153,26 @@ impl BlobStorageBackend for AzureBlobBackend {
})
}
/// Dedup settle path: PUT unconditionally. Content-addressed keys make
/// re-PUTs idempotent, so the `get_properties` probe
/// `put_blob_from_bytes` pays is a pure extra round-trip on every NEW
/// chunk (2 RTTs -> 1, benches/S3-PUT.md — same shape as S3).
fn put_blob_from_bytes_unsynced(
&self,
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let client = self.blob_client(&hash);
let size = data.len() as u64;
client.put_block_blob(data).await.map_err(|e| {
DomainError::internal_error("Azure", format!("Failed to upload blob {hash}: {e}"))
})?;
Ok(size)
})
}
fn get_blob_stream(
&self,
hash: &str,
@@ -147,29 +182,46 @@ impl BlobStorageBackend for AzureBlobBackend {
Box::pin(async move {
let client = self.blob_client(&hash);
let mut result_data: Vec<u8> = Vec::new();
let mut stream = client.get().into_stream();
while let Some(response) = stream.next().await {
let response = response.map_err(|e| {
DomainError::new(
// The old implementation drained the ENTIRE blob into one
// `Vec<u8>` before yielding a single mega-chunk — whole-blob
// RAM residency per reader, and with `read_prefetch() = 8`
// up to 8 entire chunk-blobs resident at once during CDC
// reassembly. Now the SDK's page/body streams forward
// directly. The FIRST page is still awaited eagerly so a
// missing blob surfaces as the same up-front NotFound the
// old code produced; later pages/chunks map to io::Error
// items like every other backend's stream.
let mut pages = client.get().into_stream();
let first = match pages.next().await {
Some(Ok(response)) => response,
Some(Err(e)) => {
return Err(DomainError::new(
ErrorKind::NotFound,
"Azure",
format!("Failed to get blob {hash}: {e}"),
)
})?;
let mut body = response.data;
while let Some(chunk) = body.next().await {
let chunk = chunk.map_err(|e| {
DomainError::internal_error("Azure", format!("Stream read error: {e}"))
})?;
result_data.extend_from_slice(&chunk);
));
}
}
None => {
let empty: BlobStream =
Box::pin(futures::stream::once(async move { Ok(Bytes::new()) }));
return Ok(empty);
}
};
let stream: BlobStream = Box::pin(futures::stream::once(async move {
Ok(Bytes::from(result_data))
}));
let first_body = first.data.map(|chunk| {
chunk.map_err(|e| std::io::Error::other(format!("Stream read error: {e}")))
});
let tail = pages
.map(|page| match page {
Ok(response) => Ok(response.data.map(|chunk| {
chunk.map_err(|e| std::io::Error::other(format!("Stream read error: {e}")))
})),
Err(e) => Err(std::io::Error::other(format!(
"Failed to get blob page: {e}"
))),
})
.try_flatten();
let stream: BlobStream = Box::pin(first_body.chain(tail));
Ok(stream)
})
}
@@ -190,32 +242,42 @@ impl BlobStorageBackend for AzureBlobBackend {
None => azure_core::request_options::Range::new(start, u64::MAX),
};
let mut result_data: Vec<u8> = Vec::new();
let mut stream = client.get().range(range).into_stream();
while let Some(response) = stream.next().await {
let response = response.map_err(|e| {
DomainError::new(
// Same forwarding shape as `get_blob_stream` — a ranged read
// doubly so: the caller explicitly asked NOT to pay for the
// whole blob, yet the old code buffered the full range.
let mut pages = client.get().range(range).into_stream();
let first = match pages.next().await {
Some(Ok(response)) => response,
Some(Err(e)) => {
return Err(DomainError::new(
ErrorKind::NotFound,
"Azure",
format!("Failed to get blob range {hash}: {e}"),
)
})?;
let mut body = response.data;
while let Some(chunk) = body.next().await {
let chunk = chunk.map_err(|e| {
DomainError::internal_error(
"Azure",
format!("Stream range read error: {e}"),
)
})?;
result_data.extend_from_slice(&chunk);
));
}
}
None => {
let empty: BlobStream =
Box::pin(futures::stream::once(async move { Ok(Bytes::new()) }));
return Ok(empty);
}
};
let stream: BlobStream = Box::pin(futures::stream::once(async move {
Ok(Bytes::from(result_data))
}));
let first_body = first.data.map(|chunk| {
chunk.map_err(|e| std::io::Error::other(format!("Stream range read error: {e}")))
});
let tail = pages
.map(|page| match page {
Ok(response) => Ok(response.data.map(|chunk| {
chunk.map_err(|e| {
std::io::Error::other(format!("Stream range read error: {e}"))
})
})),
Err(e) => Err(std::io::Error::other(format!(
"Failed to get blob range page: {e}"
))),
})
.try_flatten();
let stream: BlobStream = Box::pin(first_body.chain(tail));
Ok(stream)
})
}
+246 -234
View File
@@ -10,15 +10,14 @@
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use bytes::Bytes;
use lru::LruCache;
use std::num::NonZeroUsize;
use dashmap::DashMap;
use tokio::fs;
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use tokio::sync::Mutex;
use tokio_util::io::ReaderStream;
use uuid::Uuid;
use crate::application::ports::blob_storage_ports::{
BlobStorageBackend, BlobStream, StorageHealthStatus,
@@ -50,33 +49,63 @@ struct CacheEntry {
/// A `BlobStorageBackend` decorator that adds an LRU disk cache in front of
/// a remote backend.
///
/// The index is a `moka::sync::Cache` with a byte weigher: cached reads
/// probe it lock-free (sharded, striped recency) where the previous
/// `tokio::sync::Mutex<LruCache>` serialized EVERY cached chunk read on one
/// global async mutex — negative scaling under concurrent readers
/// (benches/ROUND12.md §B: 2.08 → 1.07 Mops/s going 1 → 2 readers on the
/// mutex; moka holds 1.7-2.4). moka also owns the byte budget: eviction by
/// weighted size replaces the manual `current_size` counter +
/// `collect_evictions` sweep, and the eviction listener unlinks the evicted
/// `.blob` (only on size-eviction — a Replaced entry shares its file with
/// the replacement, and Explicit invalidations unlink at their call site).
pub struct CachedBlobBackend {
inner: Arc<dyn BlobStorageBackend>,
cache_dir: PathBuf,
max_cache_bytes: u64,
index: Arc<Mutex<LruCache<String, CacheEntry>>>,
current_size: Arc<AtomicU64>,
index: moka::sync::Cache<String, CacheEntry>,
/// Per-hash single-flight gates for cache misses. K concurrent cold
/// readers of one blob (e.g. a video player's parallel Range probes)
/// used to each download the FULL blob from the remote backend — and
/// race their writes on one shared `.tmp` path. The gate coalesces
/// them onto one fetch; waiters re-check the cache and serve locally
/// (16 fetches -> 1, benches/BLOB-CACHE.md).
inflight: Arc<DashMap<String, Arc<Mutex<()>>>>,
}
fn cached_path_in(cache_dir: &Path, hash: &str) -> PathBuf {
let prefix = &hash[..2.min(hash.len())];
cache_dir.join(prefix).join(format!("{hash}.blob"))
}
impl CachedBlobBackend {
/// Create a new cached backend wrapping `inner`.
pub fn new(inner: Arc<dyn BlobStorageBackend>, config: &BlobCacheConfig) -> Self {
let listener_dir = config.cache_dir.clone();
Self {
inner,
cache_dir: config.cache_dir.clone(),
max_cache_bytes: config.max_cache_bytes,
// Capacity is essentially unbounded — eviction is by byte budget, not count.
index: Arc::new(Mutex::new(LruCache::new(
NonZeroUsize::new(1_000_000).unwrap(),
))),
current_size: Arc::new(AtomicU64::new(0)),
index: moka::sync::Cache::builder()
.weigher(|_k: &String, e: &CacheEntry| e.size.clamp(1, u32::MAX as u64) as u32)
.max_capacity(config.max_cache_bytes)
.eviction_listener(move |hash: Arc<String>, _entry, cause| {
// Size-evicted blobs lose their on-disk file here (the
// sweep `collect_evictions` used to do). A quick unlink
// on the inserting task's thread, off the hot get path.
if cause == moka::notification::RemovalCause::Size {
let _ = std::fs::remove_file(cached_path_in(&listener_dir, &hash));
}
})
.build(),
inflight: Arc::new(DashMap::new()),
}
}
/// Path where a blob is cached locally.
fn cached_path(&self, hash: &str) -> PathBuf {
let prefix = &hash[..2.min(hash.len())];
self.cache_dir.join(prefix).join(format!("{hash}.blob"))
cached_path_in(&self.cache_dir, hash)
}
}
@@ -87,14 +116,24 @@ impl BlobStorageBackend for CachedBlobBackend {
let inner = self.inner.clone();
let cache_dir = self.cache_dir.clone();
let index = self.index.clone();
let current_size = self.current_size.clone();
Box::pin(async move {
inner.initialize().await?;
// Create cache dir structure (256 prefix dirs)
// Create the cache dir AND its 256 {00..ff} shard dirs up front
// (mirroring LocalBlobBackend::initialize), so the write paths never
// pay a per-chunk `create_dir_all` on an already-existing shard — a
// ~45 µs mkdirat(EEXIST)+stat+blocking-dispatch removed per cache
// write on cached-remote deployments (benches/ROUND26.md §D1).
fs::create_dir_all(&cache_dir).await.map_err(|e| {
DomainError::internal_error("BlobCache", format!("mkdir cache_dir: {e}"))
})?;
for prefix in &crate::infrastructure::services::local_blob_backend::HEX_PREFIXES {
fs::create_dir_all(cache_dir.join(prefix))
.await
.map_err(|e| {
DomainError::internal_error("BlobCache", format!("mkdir cache shard: {e}"))
})?;
}
// Scan existing cache to rebuild index. Collect entries WITHOUT
// holding the index lock — a large cache directory walk must not
@@ -120,14 +159,13 @@ impl BlobStorageBackend for CachedBlobBackend {
}
}
}
// Bulk-insert the rebuilt index under a single brief lock.
{
let mut idx = index.lock().await;
for (stem, size) in entries {
idx.put(stem, CacheEntry { size });
}
// Rebuild the index; if the restored set exceeds the byte
// budget, moka trims it (and the eviction listener unlinks the
// trimmed files) — the old index carried the excess until the
// next insert.
for (stem, size) in entries {
index.insert(stem, CacheEntry { size });
}
current_size.store(total_bytes, Ordering::Relaxed);
tracing::info!(
"Blob cache initialized: {} bytes in cache at {}",
total_bytes,
@@ -142,21 +180,28 @@ impl BlobStorageBackend for CachedBlobBackend {
hash: &str,
source_path: &Path,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let inner = self.inner.clone();
let hash = hash.to_string();
let source = source_path.to_path_buf();
let self_ref = CachedRef {
cache_dir: self.cache_dir.clone(),
max_cache_bytes: self.max_cache_bytes,
index: self.index.clone(),
current_size: self.current_size.clone(),
};
Box::pin(async move {
// Write to inner backend
let bytes = inner.put_blob(&hash, &source).await?;
// Also cache locally (best-effort)
let _ = self_ref.insert_into_cache_static(&hash, &source).await;
Ok(bytes)
// Cache FIRST: every inner backend consumes the source file
// (local renames it, S3/Azure delete it after upload), so the
// old populate-after-put ordering failed 100% of the time and
// the first read after a whole-file put paid a full remote
// re-download (the ROUND11 deferred correctness note; fix
// gated in benches/ROUND12.md §B).
let cached = self.insert_into_cache(&hash, &source).await.is_ok();
match self.inner.put_blob(&hash, &source).await {
Ok(bytes) => Ok(bytes),
Err(e) => {
// Never serve a blob the backend rejected: drop the
// just-inserted cache entry + file.
if cached {
self.index.invalidate(&hash);
let _ = fs::remove_file(self.cached_path(&hash)).await;
}
Err(e)
}
}
})
}
@@ -165,68 +210,68 @@ impl BlobStorageBackend for CachedBlobBackend {
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let inner = self.inner.clone();
let hash = hash.to_string();
let self_ref = CachedRef {
cache_dir: self.cache_dir.clone(),
max_cache_bytes: self.max_cache_bytes,
index: self.index.clone(),
current_size: self.current_size.clone(),
};
Box::pin(async move {
let size = inner.put_blob_from_bytes(&hash, data.clone()).await?;
// Also cache locally (best-effort): write bytes to cache path
let dest = self_ref.cached_path(&hash);
if let Some(parent) = dest.parent() {
let _ = fs::create_dir_all(parent).await;
}
let _ = fs::write(&dest, &data).await;
let data_len = data.len() as u64;
let mut idx = self_ref.index.lock().await;
if let Some(old) = idx.put(hash, CacheEntry { size: data_len }) {
self_ref.current_size.fetch_sub(old.size, Ordering::Relaxed);
}
self_ref.current_size.fetch_add(data_len, Ordering::Relaxed);
let size = self.inner.put_blob_from_bytes(&hash, data.clone()).await?;
self.cache_bytes_write_through(hash, &data).await;
Ok(size)
})
}
// Without this override the trait default would re-route the CDC chunk
// write through `put_blob_from_bytes` above, whose inner (synced) call
// pays the remote exists-probe per chunk. The local write-through cache
// population is kept identical — post-upload readers (thumbnail/EXIF/
// face hooks) hit the cache instead of re-fetching from the remote.
fn put_blob_from_bytes_unsynced(
&self,
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_string();
Box::pin(async move {
let size = self
.inner
.put_blob_from_bytes_unsynced(&hash, data.clone())
.await?;
self.cache_bytes_write_through(hash, &data).await;
Ok(size)
})
}
// The durability barrier must reach the backend that buffered the
// unsynced writes; the local cache copy is disposable and needs none.
fn sync_blobs(
&self,
hashes: &[String],
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
self.inner.sync_blobs(hashes)
}
fn get_blob_stream(
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<BlobStream, DomainError>> + Send + '_>>
{
let hash = hash.to_string();
let cached = self.cached_path(&hash);
let index = self.index.clone();
let inner = self.inner.clone();
let cache_dir = self.cache_dir.clone();
let max_cache_bytes = self.max_cache_bytes;
let current_size = self.current_size.clone();
Box::pin(async move {
// Check cache presence (and bump LRU recency) under a brief lock,
// then release it BEFORE touching the filesystem so concurrent
// readers don't serialize behind a single open() syscall.
if index.lock().await.get(&hash).is_some() {
// Lock-free cache probe (bumps moka recency) — the old shape
// took the one global async mutex here on EVERY cached chunk
// read, and cloned `cache_dir` per hit for a miss-only struct.
if self.index.get(&hash).is_some() {
let cached = self.cached_path(&hash);
if let Ok(file) = fs::File::open(&cached).await {
let stream: BlobStream =
Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE));
return Ok(stream);
}
// Cache entry stale (file vanished) — drop it from the index.
if let Some(entry) = index.lock().await.pop(&hash) {
current_size.fetch_sub(entry.size, Ordering::Relaxed);
}
self.index.invalidate(&hash);
}
// Cache miss — fetch from inner, spool to cache
let self_ref = CachedRef {
cache_dir,
max_cache_bytes,
index: index.clone(),
current_size: current_size.clone(),
};
let dest = self_ref.fetch_and_cache_static(&hash, &*inner).await?;
// Cache miss — fetch from inner (single-flight), spool to cache
let cached = self.cached_path(&hash);
let dest = self.fetch_and_cache_singleflight(&hash, &cached).await?;
let file = fs::File::open(&dest).await.map_err(|e| {
DomainError::internal_error("BlobCache", format!("re-open cached: {e}"))
})?;
@@ -243,17 +288,11 @@ impl BlobStorageBackend for CachedBlobBackend {
) -> Pin<Box<dyn std::future::Future<Output = Result<BlobStream, DomainError>> + Send + '_>>
{
let hash = hash.to_string();
let cached = self.cached_path(&hash);
let index = self.index.clone();
let inner = self.inner.clone();
let cache_dir = self.cache_dir.clone();
let max_cache_bytes = self.max_cache_bytes;
let current_size = self.current_size.clone();
Box::pin(async move {
// Check cache presence (and bump LRU recency) under a brief lock,
// then release it BEFORE the open()/seek() syscalls so concurrent
// range readers don't serialize behind the index mutex.
if index.lock().await.get(&hash).is_some() {
// Lock-free cache probe (bumps moka recency); the filesystem is
// only touched after the probe, as before.
if self.index.get(&hash).is_some() {
let cached = self.cached_path(&hash);
if let Ok(mut file) = fs::File::open(&cached).await {
file.seek(std::io::SeekFrom::Start(start))
.await
@@ -266,19 +305,14 @@ impl BlobStorageBackend for CachedBlobBackend {
Box::pin(ReaderStream::with_capacity(limited, STREAM_CHUNK_SIZE));
return Ok(stream);
}
if let Some(entry) = index.lock().await.pop(&hash) {
current_size.fetch_sub(entry.size, Ordering::Relaxed);
}
self.index.invalidate(&hash);
}
// Cache miss — fetch full blob into cache, then serve range
let self_ref = CachedRef {
cache_dir,
max_cache_bytes,
index: index.clone(),
current_size: current_size.clone(),
};
let dest = self_ref.fetch_and_cache_static(&hash, &*inner).await?;
// Cache miss — fetch full blob into cache (single-flight: a
// player's parallel cold Range probes coalesce onto ONE remote
// download), then serve the range locally.
let cached = self.cached_path(&hash);
let dest = self.fetch_and_cache_singleflight(&hash, &cached).await?;
let mut file = fs::File::open(&dest)
.await
.map_err(|e| DomainError::internal_error("BlobCache", format!("re-open: {e}")))?;
@@ -297,19 +331,13 @@ impl BlobStorageBackend for CachedBlobBackend {
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
let inner = self.inner.clone();
let hash = hash.to_string();
let cached = self.cached_path(&hash);
let index = self.index.clone();
let current_size = self.current_size.clone();
Box::pin(async move {
inner.delete_blob(&hash).await?;
// Remove from cache — drop the index lock before the unlink()
// syscall so deletes don't serialize concurrent cache lookups.
if let Some(entry) = index.lock().await.pop(&hash) {
current_size.fetch_sub(entry.size, Ordering::Relaxed);
}
let _ = fs::remove_file(&cached).await;
self.inner.delete_blob(&hash).await?;
// Explicit invalidation unlinks here (the eviction listener
// only unlinks size-evictions).
self.index.invalidate(&hash);
let _ = fs::remove_file(self.cached_path(&hash)).await;
Ok(())
})
}
@@ -318,18 +346,13 @@ impl BlobStorageBackend for CachedBlobBackend {
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<bool, DomainError>> + Send + '_>> {
let inner = self.inner.clone();
let hash = hash.to_string();
let index = self.index.clone();
Box::pin(async move {
// Check cache first (fast)
{
let mut idx = index.lock().await;
if idx.get(&hash).is_some() {
return Ok(true);
}
// Check cache first (fast, lock-free)
if self.index.get(&hash).is_some() {
return Ok(true);
}
inner.blob_exists(&hash).await
self.inner.blob_exists(&hash).await
})
}
@@ -337,23 +360,17 @@ impl BlobStorageBackend for CachedBlobBackend {
&self,
hash: &str,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let inner = self.inner.clone();
let hash = hash.to_string();
let index = self.index.clone();
let cached = self.cached_path(&hash);
Box::pin(async move {
// Check cache
{
let mut idx = index.lock().await;
if let Some(entry) = idx.get(&hash) {
return Ok(entry.size);
}
// Check cache (lock-free)
if let Some(entry) = self.index.get(&hash) {
return Ok(entry.size);
}
// Fallback to cached file on disk (in case index was lost)
if let Ok(meta) = fs::metadata(&cached).await {
if let Ok(meta) = fs::metadata(self.cached_path(&hash)).await {
return Ok(meta.len());
}
inner.blob_size(&hash).await
self.inner.blob_size(&hash).await
})
}
@@ -362,19 +379,18 @@ impl BlobStorageBackend for CachedBlobBackend {
) -> Pin<
Box<dyn std::future::Future<Output = Result<StorageHealthStatus, DomainError>> + Send + '_>,
> {
let inner = self.inner.clone();
let cache_dir = self.cache_dir.clone();
let current_size = self.current_size.clone();
let max_bytes = self.max_cache_bytes;
Box::pin(async move {
let mut status = inner.health_check().await?;
let used = current_size.load(Ordering::Relaxed);
let mut status = self.inner.health_check().await?;
// Flush moka's pending maintenance so the reported byte count
// is current (rare admin path — the cost is fine here).
self.index.run_pending_tasks();
let used = self.index.weighted_size();
status.message = format!(
"{} | Cache: {}/{} bytes used at {}",
status.message,
used,
max_bytes,
cache_dir.display()
self.max_cache_bytes,
self.cache_dir.display()
);
status.backend_type = format!("cached({})", status.backend_type);
Ok(status)
@@ -398,54 +414,56 @@ impl BlobStorageBackend for CachedBlobBackend {
}
}
// ── Helper struct for owned references in async closures ───────────
// ── Cache internals (miss path + population) ───────────────────────
/// Cloneable set of cache internals — avoids borrow issues in boxed futures.
struct CachedRef {
cache_dir: PathBuf,
max_cache_bytes: u64,
index: Arc<Mutex<LruCache<String, CacheEntry>>>,
current_size: Arc<AtomicU64>,
}
impl CachedRef {
fn cached_path(&self, hash: &str) -> PathBuf {
let prefix = &hash[..2.min(hash.len())];
self.cache_dir.join(prefix).join(format!("{hash}.blob"))
impl CachedBlobBackend {
/// Best-effort write-through cache population shared by both blob-bytes
/// PUT paths. moka enforces the byte budget on every insert (the old
/// index deliberately skipped the eviction sweep on this path, letting
/// write bursts overshoot the budget until the next read-miss insert).
async fn cache_bytes_write_through(&self, hash: String, data: &Bytes) {
// The shard dir was created at initialize() — no per-write create_dir_all
// (benches/ROUND26.md §D1).
let dest = self.cached_path(&hash);
let _ = fs::write(&dest, data).await;
let data_len = data.len() as u64;
self.index.insert(hash, CacheEntry { size: data_len });
}
/// Pop LRU entries until the cache is back within its byte budget,
/// returning the on-disk paths of the evicted blobs.
///
/// Only the in-memory index is touched here (atomic counter + LRU map);
/// the caller MUST unlink the returned paths AFTER releasing the index
/// lock so the `remove_file` syscalls never run while the mutex is held.
fn collect_evictions(&self, idx: &mut LruCache<String, CacheEntry>) -> Vec<PathBuf> {
let mut victims = Vec::new();
while self.current_size.load(Ordering::Relaxed) > self.max_cache_bytes {
if let Some((evicted_hash, evicted_entry)) = idx.pop_lru() {
self.current_size
.fetch_sub(evicted_entry.size, Ordering::Relaxed);
victims.push(self.cached_path(&evicted_hash));
} else {
break;
}
}
victims
}
async fn insert_into_cache_static(
/// Single-flight wrapper around [`Self::fetch_and_cache`]: the first
/// caller for a hash becomes the leader and downloads; concurrent
/// callers queue on the per-hash gate, then re-check the cache and serve
/// the leader's file without touching the remote backend. Errors are not
/// cached — the gate entry is dropped, so the next caller retries.
async fn fetch_and_cache_singleflight(
&self,
hash: &str,
source_path: &Path,
) -> Result<(), DomainError> {
let dest = self.cached_path(hash);
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent).await.map_err(|e| {
DomainError::internal_error("BlobCache", format!("mkdir failed: {e}"))
})?;
cached: &Path,
) -> Result<PathBuf, DomainError> {
let gate = self
.inflight
.entry(hash.to_string())
.or_insert_with(|| Arc::new(Mutex::new(())))
.clone();
let _guard = gate.lock().await;
// Re-check under the gate: if we queued behind the leader, the blob
// is on disk now and this turns into a local open.
if self.index.get(hash).is_some() && fs::metadata(cached).await.is_ok() {
return Ok(cached.to_path_buf());
}
let result = self.fetch_and_cache(hash).await;
// Drop the gate whether we succeeded or failed; a late-arriving
// caller after an error creates a fresh gate and retries the fetch.
self.inflight.remove(hash);
result
}
async fn insert_into_cache(&self, hash: &str, source_path: &Path) -> Result<(), DomainError> {
// Shard dir pre-created at initialize() (benches/ROUND26.md §D1).
let dest = self.cached_path(hash);
let size = fs::metadata(source_path)
.await
.map(|m| m.len())
@@ -455,74 +473,68 @@ impl CachedRef {
DomainError::internal_error("BlobCache", format!("cache copy failed: {e}"))
})?;
// Update the index and pick eviction victims under a single brief
// lock, then unlink the evicted files AFTER releasing it — file
// removal must not run while the index mutex is held.
let to_evict = {
let mut idx = self.index.lock().await;
if let Some(old) = idx.put(hash.to_string(), CacheEntry { size }) {
self.current_size.fetch_sub(old.size, Ordering::Relaxed);
}
self.current_size.fetch_add(size, Ordering::Relaxed);
self.collect_evictions(&mut idx)
};
for path in to_evict {
let _ = fs::remove_file(&path).await;
}
// moka enforces the byte budget; size-evicted victims are unlinked
// by the eviction listener.
self.index.insert(hash.to_string(), CacheEntry { size });
Ok(())
}
async fn fetch_and_cache_static(
&self,
hash: &str,
inner: &dyn BlobStorageBackend,
) -> Result<PathBuf, DomainError> {
let stream = inner.get_blob_stream(hash).await?;
async fn fetch_and_cache(&self, hash: &str) -> Result<PathBuf, DomainError> {
let stream = self.inner.get_blob_stream(hash).await?;
// Shard dir pre-created at initialize() (benches/ROUND26.md §D1).
let dest = self.cached_path(hash);
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent).await.map_err(|e| {
DomainError::internal_error("BlobCache", format!("mkdir failed: {e}"))
// Unique temp name: even if two fetches for one hash ever race
// (e.g. across processes sharing a cache dir), each writes its own
// inode and the rename is atomic — a torn/interleaved file can
// never land at the final path.
let tmp = dest.with_extension(format!("{}.tmp", Uuid::new_v4()));
let write_result: Result<u64, DomainError> = async {
let mut file = fs::File::create(&tmp).await.map_err(|e| {
DomainError::internal_error("BlobCache", format!("create tmp: {e}"))
})?;
}
let tmp = dest.with_extension("tmp");
let mut file = fs::File::create(&tmp)
.await
.map_err(|e| DomainError::internal_error("BlobCache", format!("create tmp: {e}")))?;
use futures::StreamExt;
let mut stream = stream;
let mut total = 0u64;
while let Some(chunk) = stream.next().await {
let bytes = chunk.map_err(|e| {
DomainError::internal_error("BlobCache", format!("stream read: {e}"))
})?;
total += bytes.len() as u64;
file.write_all(&bytes)
.await
.map_err(|e| DomainError::internal_error("BlobCache", format!("write: {e}")))?;
}
file.flush()
.await
.map_err(|e| DomainError::internal_error("BlobCache", format!("flush: {e}")))?;
drop(file);
fs::rename(&tmp, &dest)
.await
.map_err(|e| DomainError::internal_error("BlobCache", format!("rename: {e}")))?;
let to_evict = {
let mut idx = self.index.lock().await;
if let Some(old) = idx.put(hash.to_string(), CacheEntry { size: total }) {
self.current_size.fetch_sub(old.size, Ordering::Relaxed);
use futures::StreamExt;
let mut stream = stream;
let mut total = 0u64;
while let Some(chunk) = stream.next().await {
let bytes = chunk.map_err(|e| {
DomainError::internal_error("BlobCache", format!("stream read: {e}"))
})?;
total += bytes.len() as u64;
file.write_all(&bytes)
.await
.map_err(|e| DomainError::internal_error("BlobCache", format!("write: {e}")))?;
}
self.current_size.fetch_add(total, Ordering::Relaxed);
self.collect_evictions(&mut idx)
};
for path in to_evict {
let _ = fs::remove_file(&path).await;
file.flush()
.await
.map_err(|e| DomainError::internal_error("BlobCache", format!("flush: {e}")))?;
Ok(total)
}
.await;
let total = match write_result {
Ok(total) => total,
Err(e) => {
// Unique tmp names never get overwritten by a later fetch —
// reap the partial file instead of leaking it.
let _ = fs::remove_file(&tmp).await;
return Err(e);
}
};
if let Err(e) = fs::rename(&tmp, &dest).await {
let _ = fs::remove_file(&tmp).await;
return Err(DomainError::internal_error(
"BlobCache",
format!("rename: {e}"),
));
}
// moka enforces the byte budget; size-evicted victims are unlinked
// by the eviction listener.
self.index
.insert(hash.to_string(), CacheEntry { size: total });
Ok(dest)
}
@@ -514,6 +514,17 @@ impl ChunkedUploadService {
Ok(())
}
/// Alloc-free owner compare for the per-chunk hot path: the caller's
/// `Uuid` is stack-encoded (hyphenated, the format sessions store) —
/// `prepare_chunk`/`commit_chunk` used to pay a `Uuid::to_string` each
/// plus a dedicated `verify_session_owner` map lookup per chunk
/// (benches/ROUND12.md §M5, 1.28x / −2 allocs per chunk).
#[inline]
fn owner_matches(session_user_id: &str, user_id: Uuid) -> bool {
let mut buf = [0u8; 36];
session_user_id == user_id.hyphenated().encode_lower(&mut buf) as &str
}
/// Create a new upload session (persists `session.json` + empty `progress.bin`)
async fn create_session_inner(
&self,
@@ -617,9 +628,8 @@ impl ChunkedUploadService {
user_id: Uuid,
chunk_index: usize,
) -> Result<(PathBuf, usize), DomainError> {
self.verify_session_owner(upload_id, &user_id.to_string())
.map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))?;
// Single map lookup: the owner gate rides the same guard (same
// anti-enum not-found for unknown session and foreign session).
let session = self.sessions.get(upload_id).ok_or_else(|| {
DomainError::new(
ErrorKind::NotFound,
@@ -627,6 +637,13 @@ impl ChunkedUploadService {
format!("Upload session not found: {}", upload_id),
)
})?;
if !Self::owner_matches(&session.user_id, user_id) {
return Err(DomainError::new(
ErrorKind::NotFound,
"ChunkedUpload",
format!("Upload session not found: {}", upload_id),
));
}
if chunk_index >= session.chunks.len() {
return Err(DomainError::new(
@@ -678,20 +695,23 @@ impl ChunkedUploadService {
computed_checksum: Option<String>,
expected_checksum: Option<String>,
) -> Result<ChunkUploadResponseDto, DomainError> {
self.verify_session_owner(upload_id, &user_id.to_string())
.map_err(|e| DomainError::new(ErrorKind::NotFound, "ChunkedUpload", e))?;
// Re-fetch chunk metadata under fresh lock — guards against the
// (vanishingly unlikely) case of a session expiry / cancellation
// racing with the write.
// Owner gate folded into the metadata read below — one lookup
// instead of two, same anti-enum not-found semantics.
let (chunk_path, expected_size, persist_path) = {
let session = self.sessions.get(upload_id).ok_or_else(|| {
DomainError::new(
ErrorKind::NotFound,
"ChunkedUpload",
"Session disappeared".to_string(),
format!("Upload session not found: {}", upload_id),
)
})?;
if !Self::owner_matches(&session.user_id, user_id) {
return Err(DomainError::new(
ErrorKind::NotFound,
"ChunkedUpload",
format!("Upload session not found: {}", upload_id),
));
}
if chunk_index >= session.chunks.len() {
return Err(DomainError::new(
ErrorKind::InvalidInput,
@@ -834,8 +854,7 @@ impl ChunkedUploadService {
let data_clone = data.clone(); // Bytes::clone is O(1) — just an Arc increment
let actual_checksum = tokio::task::spawn_blocking(move || {
use md5::{Digest, Md5};
let hash = Md5::digest(&data_clone);
hash.iter().map(|b| format!("{b:02x}")).collect::<String>()
crate::common::fmt::hex_lower(&Md5::digest(&data_clone))
})
.await
.map_err(|e| format!("MD5 checksum task failed: {e}"))?;
File diff suppressed because it is too large Load Diff
@@ -30,7 +30,7 @@
use std::path::{Path, PathBuf};
use std::pin::Pin;
use aes_gcm::aead::{Aead, AeadInPlace, KeyInit, OsRng};
use aes_gcm::aead::{AeadInPlace, KeyInit, OsRng};
use aes_gcm::{AeadCore, Aes256Gcm, Nonce};
use bytes::Bytes;
use std::sync::Arc;
@@ -44,6 +44,9 @@ use crate::domain::errors::DomainError;
/// Nonce size for AES-256-GCM (96 bits = 12 bytes).
const NONCE_SIZE: usize = 12;
/// AES-256-GCM authentication tag length appended after the ciphertext.
const TAG_SIZE: usize = 16;
/// Payloads at or above this size run crypto on the blocking pool; below
/// it the `spawn_blocking` round-trip costs more than the AES work itself.
const CRYPTO_OFFLOAD_THRESHOLD: usize = 64 * 1024;
@@ -56,7 +59,10 @@ const PLAINTEXT_EMIT_SIZE: usize = 64 * 1024;
/// `BlobStorageBackend` decorator that encrypts blobs at rest.
pub struct EncryptedBlobBackend {
inner: Arc<dyn BlobStorageBackend>,
cipher: Aes256Gcm,
/// `Arc` so the per-op `clone()` handed to `offload_crypto` closures is
/// an atomic bump instead of copying the ~240-byte expanded AES-256
/// round-key schedule on every chunk read/write.
cipher: Arc<Aes256Gcm>,
}
impl EncryptedBlobBackend {
@@ -64,7 +70,8 @@ impl EncryptedBlobBackend {
///
/// `key` must be exactly 32 bytes (AES-256).
pub fn new(inner: Arc<dyn BlobStorageBackend>, key: &[u8; 32]) -> Self {
let cipher = Aes256Gcm::new_from_slice(key).expect("AES-256 key must be 32 bytes");
let cipher =
Arc::new(Aes256Gcm::new_from_slice(key).expect("AES-256 key must be 32 bytes"));
Self { inner, cipher }
}
@@ -78,36 +85,60 @@ impl EncryptedBlobBackend {
}
/// Encrypt `data` into the on-disk layout: `[12-byte nonce][ciphertext + tag]`.
///
/// Single output buffer, mirroring the read side's in-place detached decrypt:
/// the payload is copied exactly once and encrypted in place with the tag
/// appended. The old shape let `cipher.encrypt` allocate a full ciphertext
/// `Vec` and then copied it a second time behind the nonce — one extra
/// allocation + a full-size memcpy on every encrypted chunk write
/// (benches/ROUND11.md §15; output bytes identical for a given nonce).
fn encrypt_bytes(cipher: &Aes256Gcm, data: &[u8]) -> Result<Bytes, DomainError> {
let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
let ciphertext = cipher
.encrypt(&nonce, data)
let mut out = Vec::with_capacity(NONCE_SIZE + data.len() + TAG_SIZE);
out.extend_from_slice(nonce.as_slice());
out.extend_from_slice(data);
let tag = cipher
.encrypt_in_place_detached(&nonce, b"", &mut out[NONCE_SIZE..])
.map_err(|e| DomainError::internal_error("Encryption", format!("encrypt failed: {e}")))?;
let mut encrypted = Vec::with_capacity(NONCE_SIZE + ciphertext.len());
encrypted.extend_from_slice(nonce.as_slice());
encrypted.extend_from_slice(&ciphertext);
Ok(Bytes::from(encrypted))
out.extend_from_slice(&tag);
Ok(Bytes::from(out))
}
/// Decrypt the on-disk layout `[nonce][ciphertext + tag]` **in place**.
///
/// Consumes the encrypted buffer and reuses it for the plaintext, so peak
/// RAM is one buffer — not ciphertext + plaintext side by side (which for
/// legacy whole-file blobs would double a multi-hundred-MB allocation).
/// Reuses the encrypted buffer for the plaintext, so peak RAM is one buffer —
/// not ciphertext + plaintext side by side (which for legacy whole-file blobs
/// would double a multi-hundred-MB allocation). The nonce and 16-byte GCM tag
/// are lifted to the stack, the ciphertext body is decrypted in place via the
/// detached API (mirroring the encrypt side's `encrypt_in_place_detached`), and
/// the plaintext is returned as a zero-copy `Bytes::slice` past the nonce.
///
/// The prior shape did `encrypted.split_off(NONCE_SIZE)`, which allocated a
/// fresh `Vec` and memcpy'd the entire ciphertext (up to a whole legacy blob)
/// on every decrypted read — one full-payload allocation + copy the doc comment
/// above claimed did not happen (benches/ROUND25.md §M1; ROUND11 §15 fixed only
/// the encrypt side). Output plaintext is byte-identical.
fn decrypt_bytes(cipher: &Aes256Gcm, mut encrypted: Vec<u8>) -> Result<Bytes, DomainError> {
if encrypted.len() < NONCE_SIZE {
let len = encrypted.len();
if len < NONCE_SIZE + TAG_SIZE {
return Err(DomainError::internal_error(
"Encryption",
"encrypted blob too short (missing nonce)",
"encrypted blob too short (missing nonce/tag)",
));
}
let mut ciphertext = encrypted.split_off(NONCE_SIZE); // `encrypted` keeps the nonce
let nonce = Nonce::from_slice(&encrypted);
// Nonce (first 12 bytes) and GCM tag (last 16 bytes) copied to the stack so
// the middle can be borrowed mutably for in-place decryption.
let mut nonce_buf = [0u8; NONCE_SIZE];
nonce_buf.copy_from_slice(&encrypted[..NONCE_SIZE]);
let nonce = Nonce::from_slice(&nonce_buf);
let tag = aes_gcm::aead::Tag::<Aes256Gcm>::clone_from_slice(&encrypted[len - TAG_SIZE..]);
cipher
.decrypt_in_place(nonce, b"", &mut ciphertext)
.decrypt_in_place_detached(nonce, b"", &mut encrypted[NONCE_SIZE..len - TAG_SIZE], &tag)
.map_err(|e| DomainError::internal_error("Encryption", format!("decrypt failed: {e}")))?;
Ok(Bytes::from(ciphertext))
// Plaintext now lives at `encrypted[NONCE_SIZE..len - TAG_SIZE]`; drop the
// tag and hand out a refcounted view past the nonce — no copy, no new alloc.
encrypted.truncate(len - TAG_SIZE);
Ok(Bytes::from(encrypted).slice(NONCE_SIZE..))
}
/// Run a crypto closure inline for small payloads, on the blocking pool for
@@ -126,13 +157,18 @@ where
}
/// Turn a decrypted payload into a stream of bounded, zero-copy slices.
///
/// The emit-slice iterator is handed to `stream::iter` lazily — the closure
/// owns `data` (a refcounted `Bytes`), so each `slice` is produced on demand
/// as the consumer polls, rather than eagerly `collect`ing a `Vec` of
/// ⌈len/64 KiB⌉ slice handles up front (benches/ROUND20.md §I4).
fn plaintext_stream(data: Bytes) -> BlobStream {
let len = data.len();
let slices: Vec<Result<Bytes, std::io::Error>> = (0..len)
.step_by(PLAINTEXT_EMIT_SIZE)
.map(|off| Ok(data.slice(off..len.min(off + PLAINTEXT_EMIT_SIZE))))
.collect();
Box::pin(futures::stream::iter(slices))
Box::pin(futures::stream::iter(
(0..len)
.step_by(PLAINTEXT_EMIT_SIZE)
.map(move |off| Ok(data.slice(off..len.min(off + PLAINTEXT_EMIT_SIZE)))),
))
}
impl BlobStorageBackend for EncryptedBlobBackend {
@@ -318,6 +354,13 @@ impl BlobStorageBackend for EncryptedBlobBackend {
}
/// Collect a byte stream into a single `Vec<u8>`.
///
/// Modern blobs are CDC chunks (≤ `CDC_MAX_CHUNK` + nonce/tag overhead),
/// delivered here as small reader frames — growing from `Vec::new()` paid
/// ~log₂(n) reallocations + a wasted ~0.75×-size memcpy per read. Reserving
/// one chunk's worth up front on the first frame makes the common case a
/// single allocation; legacy whole-file blobs beyond that fall back to
/// normal doubling (benches/ROUND11.md §16: 9 → 1 allocs on a 1 MiB blob).
async fn collect_stream(stream: BlobStream) -> Result<Vec<u8>, DomainError> {
use futures::StreamExt;
let mut stream = stream;
@@ -325,6 +368,14 @@ async fn collect_stream(stream: BlobStream) -> Result<Vec<u8>, DomainError> {
while let Some(chunk) = stream.next().await {
let bytes = chunk
.map_err(|e| DomainError::internal_error("Encryption", format!("stream read: {e}")))?;
if buf.capacity() == 0 {
buf.reserve(
(crate::infrastructure::services::dedup_service::CDC_MAX_CHUNK
+ NONCE_SIZE
+ TAG_SIZE)
.max(bytes.len()),
);
}
buf.extend_from_slice(&bytes);
}
Ok(buf)
+25 -12
View File
@@ -61,23 +61,13 @@ impl ExifService {
// ── Camera info ──
if let Some(field) = exif.get_field(Tag::Make, In::PRIMARY) {
let val = field
.display_value()
.to_string()
.trim_matches('"')
.trim()
.to_string();
let val = display_value_trimmed(field);
if !val.is_empty() {
meta.camera_make = Some(val);
}
}
if let Some(field) = exif.get_field(Tag::Model, In::PRIMARY) {
let val = field
.display_value()
.to_string()
.trim_matches('"')
.trim()
.to_string();
let val = display_value_trimmed(field);
if !val.is_empty() {
meta.camera_model = Some(val);
}
@@ -115,6 +105,29 @@ impl ExifService {
}
}
/// Render an EXIF field's display value, then strip surrounding quotes and
/// whitespace (the shape `Make`/`Model` want) in a SINGLE allocation.
///
/// `display_value().to_string()` is the one unavoidable allocation — the field
/// value is materialized to text. The old `…to_string().trim_matches('"')
/// .trim().to_string()` chain then threw that `String` away and allocated a
/// second time for the trimmed copy. Here the same two-stage trim is applied
/// in place on the already-owned buffer (`drain` drops the prefix, `truncate`
/// the suffix — both reuse the allocation), so a quoted `"Canon"` costs one
/// allocation instead of two.
fn display_value_trimmed(field: &exif::Field) -> String {
let mut s = field.display_value().to_string();
// Same order the old chain used: strip `"` first, then whitespace. The
// result is a contiguous subslice of `s`; capture its byte range before
// mutating the owned buffer (the borrow ends at these two reads).
let trimmed = s.trim_matches('"').trim();
let start = trimmed.as_ptr().addr() - s.as_ptr().addr();
let len = trimmed.len();
s.drain(..start);
s.truncate(len);
s
}
/// Parse EXIF datetime string "YYYY:MM:DD HH:MM:SS" into DateTime<Utc>.
fn parse_exif_datetime(s: &str) -> Option<DateTime<Utc>> {
// EXIF dates use ":" as separator for date parts
@@ -28,11 +28,35 @@ fn is_image(content_type: &str) -> bool {
content_type.starts_with("image/")
}
/// Concurrent index-task budget. Env override
/// `OXICLOUD_FACES_INDEX_CONCURRENCY`, else the effective core count —
/// each task is a full-image read + decode + ONNX inference, so more
/// permits than cores only adds RAM pressure, not throughput.
fn max_concurrent_index() -> usize {
std::env::var("OXICLOUD_FACES_INDEX_CONCURRENCY")
.ok()
.and_then(|v| v.parse().ok())
.filter(|&n: &usize| n > 0)
.unwrap_or_else(|| {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(2)
})
}
pub struct FaceIndexingService {
pool: Arc<PgPool>,
repo: Arc<FacePgRepository>,
analyzer: Arc<dyn FaceAnalyzerPort>,
blob_root: PathBuf,
/// Bounds concurrent indexing tasks. The lifecycle hooks spawn one
/// task per uploaded/copied image with no ceiling, so a bulk upload
/// used to fan out N simultaneous full-image reads + decodes +
/// inferences — peak RSS N × image size plus CPU thrash. Same
/// invariant as `ThumbnailService::decode_semaphore`: the permit is
/// acquired BEFORE the blob read, so peak memory is
/// `permits × image size` regardless of upload concurrency.
index_semaphore: Arc<tokio::sync::Semaphore>,
}
impl FaceIndexingService {
@@ -43,6 +67,7 @@ impl FaceIndexingService {
repo,
analyzer,
blob_root,
index_semaphore: Arc::new(tokio::sync::Semaphore::new(max_concurrent_index())),
}
}
@@ -60,7 +85,15 @@ impl FaceIndexingService {
let repo = self.repo.clone();
let analyzer = self.analyzer.clone();
let blob_path = self.blob_path(&blob_hash);
let semaphore = self.index_semaphore.clone();
tokio::spawn(async move {
// Queue behind the concurrency budget BEFORE touching the
// blob — excess tasks wait holding only this tiny future,
// not a decoded image.
let _permit = semaphore
.acquire_owned()
.await
.expect("face index semaphore never closes");
if delete_first {
let _ = repo.delete_faces_for_file(file_id).await;
}
@@ -182,7 +182,30 @@ impl FileContentCache {
if let Some(hit) = self.get(&cache_key).await {
return Ok(hit);
}
self.load_and_cache(cache_key, etag, content_type, load)
.await
}
/// The populate-on-miss half of [`Self::get_or_load`], with single-flight
/// coalescing but WITHOUT the leading `get` probe.
///
/// Hot read paths that have *already* probed the cache with [`Self::get`]
/// (a borrow) call this directly on the miss branch — they then build the
/// owned `cache_key` / `etag` / `content_type` (each a heap allocation)
/// only when they are actually needed to populate, so a cache HIT allocates
/// none of them (benches/ROUND29.md §B). Because the caller's own `get`
/// already counted the hit/miss, this method does not re-probe — keeping the
/// hit/miss stat counts identical to a single `get_or_load` call.
pub async fn load_and_cache<F>(
&self,
cache_key: String,
etag: Arc<str>,
content_type: Arc<str>,
load: F,
) -> Result<(Bytes, Arc<str>, Arc<str>), DomainError>
where
F: Future<Output = Result<Bytes, DomainError>>,
{
// Slow path: coalesce concurrent misses into a single `load`.
let entry = self
.cache
@@ -0,0 +1,124 @@
//! Background daemon that purges expired `storage.role_grants` rows.
//!
//! The AuthZ engine already filters expired grants out of every
//! permission check at read time (`expires_at IS NULL OR
//! expires_at > NOW()` on every `check` / `list_grants_*` path in
//! `PgAclEngine`), so expired rows never leak permission. They just
//! accumulate. This daemon garbage-collects them once per
//! [`GrantCleanupService::interval_hours`], with a grace window past
//! `expires_at` that preserves the audit / support answer to "what
//! happened to my access?" for a few weeks.
//!
//! Shape mirrors [`TrashCleanupService`] verbatim (fire-and-forget
//! `tokio::spawn`, `tokio::time::interval`, first-tick-immediate). The
//! authoritative pattern for background daemons in this codebase; see
//! the plan doc `docs/plan/` (deferred future work: fold all daemons
//! into a central `JobRegistry` that plugins can also register into).
//!
//! [`TrashCleanupService`]: crate::infrastructure::services::trash_cleanup_service::TrashCleanupService
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::time;
use tracing::{error, info};
use crate::application::ports::authorization_ports::AuthorizationEngine;
use crate::infrastructure::services::pg_acl_engine::PgAclEngine;
/// Daemon that periodically deletes expired grants.
///
/// Owns an `Arc<PgAclEngine>` (not a `dyn AuthorizationEngine`) to avoid
/// the wrapper allocation on every SQL call — the daemon is the sole
/// caller of `purge_expired_grants` outside of the admin trigger
/// endpoint, both statically dispatched.
pub struct GrantCleanupService {
authz: Arc<PgAclEngine>,
grace_days: u32,
interval_hours: u64,
}
impl GrantCleanupService {
pub fn new(authz: Arc<PgAclEngine>, grace_days: u32, interval_hours: u64) -> Self {
Self {
authz,
grace_days,
// Minimum 1 hour — matches TrashCleanupService's clamp so
// a mis-set `0` doesn't spin a hot loop.
interval_hours: interval_hours.max(1),
}
}
/// Grace period the daemon uses on its scheduled ticks. Exposed
/// for the admin trigger's default-response field.
pub fn grace_days(&self) -> u32 {
self.grace_days
}
/// Fire-and-forget the periodic purge. Never joins; killed
/// implicitly at `tokio::runtime::shutdown`.
pub async fn start_cleanup_job(self: Arc<Self>) {
let interval_hours = self.interval_hours;
let grace_days = self.grace_days;
info!(
"Starting grant-cleanup daemon: every {}h, grace = {}d",
interval_hours, grace_days
);
tokio::spawn(async move {
let mut interval = time::interval(Duration::from_secs(interval_hours * 60 * 60));
// First tick fires immediately — matches TrashCleanupService.
// Any accumulated backlog at boot gets flushed straight away.
loop {
interval.tick().await;
self.run_once().await;
}
});
}
/// One scheduled pass. Also called by the admin trigger endpoint
/// (via a shared `Arc<GrantCleanupService>` on `AppState`).
///
/// `grace_override`:
/// - `None` → use the configured grace (`self.grace_days`).
/// - `Some(n)` → override with `n`. The admin `?force=true` trigger
/// passes `Some(0)` so Hurl regressions can hit expired grants
/// without waiting the configured grace out.
pub async fn purge(&self, grace_override: Option<u32>) -> u64 {
let grace = grace_override.unwrap_or(self.grace_days);
let start = Instant::now();
match self.authz.purge_expired_grants(grace).await {
Ok(count) => {
// Audit-channel logging: bulk deletion of authorization
// rows is security-relevant enough to keep it in the
// audit stream even when the count is zero (proves the
// daemon is reachable).
info!(
target: "audit",
event = "grant_cleanup.purged",
count = count,
grace_days = grace,
elapsed_ms = start.elapsed().as_millis() as u64,
"👮🏻‍♂️ Purged {} expired grant(s) older than {} days",
count,
grace,
);
count
}
Err(e) => {
error!(
target: "audit",
event = "grant_cleanup.failed",
grace_days = grace,
error = %e,
"Grant cleanup failed"
);
0
}
}
}
/// Convenience for the scheduled loop.
async fn run_once(&self) {
let _ = self.purge(None).await;
}
}
+45 -33
View File
@@ -24,6 +24,11 @@ use crate::domain::entities::user::User;
/// Internal JWT claims structure for serialization.
/// This is the actual JWT payload structure used by jsonwebtoken crate.
///
/// `username` / `email` deserialize straight into `Arc<str>` (serde `rc`,
/// one allocation — same count as `String`) so the `TokenClaims` conversion
/// below is a plain move and the port-level claims can hand refcount bumps
/// to every consumer.
#[derive(Debug, Serialize, Deserialize)]
struct JwtClaims {
/// Subject identifier - contains the user ID
@@ -35,16 +40,23 @@ struct JwtClaims {
/// JWT unique ID for token tracking and revocation
pub jti: String,
/// Username for display and identification purposes
pub username: String,
pub username: Arc<str>,
/// User email for communication and identification
pub email: String,
pub email: Arc<str>,
/// User role for authorization checks
pub role: String,
}
impl From<JwtClaims> for TokenClaims {
fn from(claims: JwtClaims) -> Self {
// Pre-parse the subject once at decode time (amortized over the
// validation-cache TTL) so the auth middleware reads a `Copy` instead
// of re-parsing the 36-char string per request. A verified token we
// signed always carries a UUID `sub`; nil is a safe sentinel the
// middleware rejects. See benches/ROUND14.md §A3.
let sub_id = uuid::Uuid::parse_str(&claims.sub).unwrap_or_else(|_| uuid::Uuid::nil());
TokenClaims {
sub_id,
sub: claims.sub,
exp: claims.exp,
iat: claims.iat,
@@ -80,8 +92,16 @@ impl From<JwtClaims> for TokenClaims {
/// unique-token flooding.
/// - Expired tokens are never cached (decode itself rejects them first).
pub struct JwtTokenService {
/// Secret key used for signing JWT tokens
jwt_secret: String,
/// Pre-built signing key — `EncodingKey::from_secret` copies the secret
/// into a fresh buffer, so building it per `generate_access_token` call
/// paid an allocation per login/refresh for a process-invariant value.
encoding_key: EncodingKey,
/// Pre-built verification key (same rationale, on the validation-cache
/// miss path — every new token and every token once per TTL window).
decoding_key: DecodingKey,
/// Pre-built HS256 validation config — `Validation::new` allocates a
/// `HashSet{"exp"}` + algorithm `Vec` on every call otherwise.
validation: Validation,
/// Expiration time for access tokens in seconds
access_token_expiry: i64,
/// Expiration time for refresh tokens in seconds
@@ -125,7 +145,9 @@ impl JwtTokenService {
);
Self {
jwt_secret,
encoding_key: EncodingKey::from_secret(jwt_secret.as_bytes()),
decoding_key: DecodingKey::from_secret(jwt_secret.as_bytes()),
validation: Validation::new(Algorithm::HS256),
access_token_expiry: access_token_expiry_secs,
refresh_token_expiry: refresh_token_expiry_secs,
validation_cache,
@@ -169,9 +191,9 @@ impl TokenServicePort for JwtTokenService {
exp: now + self.access_token_expiry,
iat: now,
jti: Uuid::new_v4().to_string(),
username: user.username().unwrap_or("").to_string(),
email: user.email().to_string(),
role: format!("{}", user.role()),
username: Arc::from(user.username().unwrap_or("")),
email: Arc::from(user.email()),
role: user.role().as_str().to_string(),
};
// Log JWT claims for debugging
@@ -182,12 +204,7 @@ impl TokenServicePort for JwtTokenService {
claims.iat
);
encode(
&Header::default(),
&claims,
&EncodingKey::from_secret(self.jwt_secret.as_bytes()),
)
.map_err(|e| {
encode(&Header::default(), &claims, &self.encoding_key).map_err(|e| {
tracing::error!("Error generating token: {}", e);
DomainError::new(
ErrorKind::InternalError,
@@ -217,23 +234,18 @@ impl TokenServicePort for JwtTokenService {
// ── 2. Slow-path: full HMAC-SHA256 verification ─────────
self.cache_misses.fetch_add(1, Ordering::Relaxed);
let validation = Validation::new(Algorithm::HS256);
let token_data = decode::<JwtClaims>(
token,
&DecodingKey::from_secret(self.jwt_secret.as_bytes()),
&validation,
)
.map_err(|e| match e.kind() {
jsonwebtoken::errors::ErrorKind::ExpiredSignature => {
DomainError::new(ErrorKind::AccessDenied, "TokenService", "Token expired")
}
_ => DomainError::new(
ErrorKind::AccessDenied,
"TokenService",
format!("Invalid token: {}", e),
),
})?;
let token_data = decode::<JwtClaims>(token, &self.decoding_key, &self.validation).map_err(
|e| match e.kind() {
jsonwebtoken::errors::ErrorKind::ExpiredSignature => {
DomainError::new(ErrorKind::AccessDenied, "TokenService", "Token expired")
}
_ => DomainError::new(
ErrorKind::AccessDenied,
"TokenService",
format!("Invalid token: {}", e),
),
},
)?;
let claims = Arc::new(TokenClaims::from(token_data.claims));
@@ -301,8 +313,8 @@ mod tests {
.validate_token(&token)
.expect("Should validate token");
assert_eq!(claims.sub, user.id().to_string());
assert_eq!(Some(claims.username.as_str()), user.username());
assert_eq!(claims.email, user.email());
assert_eq!(Some(&*claims.username), user.username());
assert_eq!(&*claims.email, user.email());
}
#[test]
@@ -128,20 +128,43 @@ async fn fsync_paths_parallel(paths: Vec<PathBuf>, strict: bool) -> Result<(), D
/// (fsync now vs. deferred batch sync), or `None` when the blob already
/// existed (idempotent skip — content-addressed, so identical by definition).
async fn write_blob_bytes(blob_path: &Path, data: &Bytes) -> Result<Option<File>, DomainError> {
if fs::try_exists(blob_path).await.unwrap_or(false) {
return Ok(None);
}
let mut file = fs::File::create(blob_path).await.map_err(|e| {
DomainError::internal_error("Blob", format!("Failed to create blob file: {}", e))
})?;
// One atomic O_CREAT|O_EXCL open replaces the old stat-then-create pair:
// `AlreadyExists` IS the idempotent skip (content-addressed names mean an
// existing file has identical content), saving a syscall + a blocking-pool
// dispatch on every new chunk of every upload.
let mut file = match fs::File::options()
.write(true)
.create_new(true)
.open(blob_path)
.await
{
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => return Ok(None),
Err(e) => {
return Err(DomainError::internal_error(
"Blob",
format!("Failed to create blob file: {}", e),
));
}
};
file.write_all(data).await.map_err(|e| {
DomainError::internal_error("Blob", format!("Failed to write blob from bytes: {}", e))
})?;
Ok(Some(file))
}
/// Bench-only public wrapper (feature = "bench") over the private chunk
/// writer so `examples/bench_storage_micro.rs` can A/B the open strategy.
#[cfg(feature = "bench")]
pub async fn write_blob_bytes_for_bench(
blob_path: &Path,
data: &Bytes,
) -> Result<Option<File>, DomainError> {
write_blob_bytes(blob_path, data).await
}
/// Compile-time lookup table for the 256 two-digit lowercase hex prefixes ("00"…"ff").
static HEX_PREFIXES: [&str; 256] = [
pub(crate) static HEX_PREFIXES: [&str; 256] = [
"00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0a", "0b", "0c", "0d", "0e", "0f",
"10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "1a", "1b", "1c", "1d", "1e", "1f",
"20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "2a", "2b", "2c", "2d", "2e", "2f",
@@ -68,7 +68,24 @@ impl LoginLockoutService {
fn key(username: &str, client_ip: &str) -> String {
// `|` is not valid in either a username or an IP literal so it makes
// the username/ip boundary unambiguous.
format!("{}|{}", username.to_lowercase(), client_ip)
//
// The lowercased composite is written into ONE pre-sized buffer instead
// of the `to_lowercase()` (alloc) + `format!` (alloc) two-step. App
// passwords authenticate with an already-lowercase ASCII username in
// ~all traffic, so the fast branch covers it; the rare non-ASCII branch
// keeps `str::to_lowercase` for exact Unicode (e.g. final-sigma)
// semantics. Byte-identical key either way (benches/ROUND29.md §D).
if username.is_ascii() {
let mut k = String::with_capacity(username.len() + 1 + client_ip.len());
for &b in username.as_bytes() {
k.push(b.to_ascii_lowercase() as char);
}
k.push('|');
k.push_str(client_ip);
k
} else {
format!("{}|{}", username.to_lowercase(), client_ip)
}
}
/// Check whether the (account, IP) pair is currently locked.
@@ -96,22 +96,33 @@ impl MediaMetadataService {
}
if Self::is_image_file(mime_type) {
// ONE disk read: kamadak needs the full buffer anyway, and
// nom-exif 3.6+ parses from in-RAM bytes zero-copy
// (`MediaSource::from_memory` over the same allocation). This
// path used to re-open the file 1-2 more times — nom-exif's
// `read_exif(path)` plus a `read_track(path)` fallback for
// date-less images (2-3 opens per image, benches/ROUND12.md §M4:
// 1.44x warm geomean, 2-3x cold-cache).
let buf = std::fs::read(path).ok()?;
// Rich EXIF (GPS / camera / orientation / dimensions + naive date)
// from the proven kamadak extractor.
let kamadak = std::fs::read(path)
.ok()
.and_then(|b| ExifService::extract(&b));
let kamadak = ExifService::extract(&buf);
// nom-exif complements kamadak: a timezone-correct capture date and,
// crucially, the date + GPS for files kamadak rejects outright
// ("Unexpected next IFD"), where `kamadak` is None and the GPS would
// otherwise be lost. See `merge_image_metadata`.
merge_image_metadata(kamadak, read_nom_exif(path))
let bytes = bytes::Bytes::from(buf);
merge_image_metadata(kamadak, read_nom_exif_from_bytes(&bytes))
} else if Self::is_video_file(mime_type) {
// Videos carry no EXIF — pull the container creation time only.
read_nom_exif(path).captured_at.map(|dt| ExifMetadata {
captured_at: Some(dt),
..Default::default()
})
// Single open + header sniff; the old shape opened twice (a
// doomed `read_exif` sniff, then `read_track`).
read_nom_exif_video(path)
.captured_at
.map(|dt| ExifMetadata {
captured_at: Some(dt),
..Default::default()
})
} else {
None
}
@@ -375,34 +386,49 @@ struct NomExif {
/// carries `OffsetTimeOriginal` (or a tz-aware container time); otherwise the
/// naive wall-clock is interpreted as UTC. Either way it is converted to a true
/// UTC instant. GPS is returned as signed decimal degrees.
fn read_nom_exif(path: &Path) -> NomExif {
use nom_exif::{EntryValue, ExifTag, TrackInfoTag, read_exif, read_track};
fn nom_to_utc(ev: &nom_exif::EntryValue) -> Option<DateTime<Utc>> {
let edt = ev.as_datetime()?;
let utc0 = FixedOffset::east_opt(0)?;
Some(edt.or_offset(utc0).with_timezone(&Utc))
}
// Captures nothing → `Copy`, so it can be reused across the calls below.
let to_utc = |ev: &EntryValue| -> Option<DateTime<Utc>> {
let edt = ev.as_datetime()?;
let utc0 = FixedOffset::east_opt(0)?;
Some(edt.or_offset(utc0).with_timezone(&Utc))
};
fn nom_fill_from_exif(exif: &nom_exif::Exif, out: &mut NomExif) {
use nom_exif::ExifTag;
out.captured_at = exif
.get(ExifTag::DateTimeOriginal)
.and_then(nom_to_utc)
.or_else(|| exif.get(ExifTag::CreateDate).and_then(nom_to_utc));
if let Some(gps) = exif.gps_info() {
out.latitude = gps.latitude_decimal();
out.longitude = gps.longitude_decimal();
}
}
/// Image arm: nom-exif fed from the buffer the kamadak pass already read —
/// `MediaSource::from_memory` shares the `Bytes` refcount, so this re-parses
/// without touching the disk again (the old shape re-opened the file once,
/// plus a second time for date-less images). The track fallback stays (fed
/// from the same bytes): it covers MIME-mislabeled rows whose actual
/// container is a video — the only case where it ever produced a date.
fn read_nom_exif_from_bytes(bytes: &bytes::Bytes) -> NomExif {
use nom_exif::{MediaParser, MediaSource, TrackInfoTag};
let mut out = NomExif::default();
let mut parser = MediaParser::new();
// Images: EXIF DateTimeOriginal → DateTimeDigitized (CreateDate), plus GPS.
if let Ok(exif) = read_exif(path) {
out.captured_at = exif
.get(ExifTag::DateTimeOriginal)
.and_then(to_utc)
.or_else(|| exif.get(ExifTag::CreateDate).and_then(to_utc));
if let Some(gps) = exif.gps_info() {
out.latitude = gps.latitude_decimal();
out.longitude = gps.longitude_decimal();
}
if let Ok(ms) = MediaSource::from_memory(bytes.clone())
&& let Ok(iter) = parser.parse_exif(ms)
{
let exif: nom_exif::Exif = iter.into();
nom_fill_from_exif(&exif, &mut out);
}
// Videos / audio containers (mov/mp4/mkv): track creation time.
if out.captured_at.is_none()
&& let Ok(track) = read_track(path)
&& let Some(dt) = track.get(TrackInfoTag::CreateDate).and_then(to_utc)
&& let Ok(ms) = MediaSource::from_memory(bytes.clone())
&& let Ok(track) = parser.parse_track(ms)
&& let Some(dt) = track.get(TrackInfoTag::CreateDate).and_then(nom_to_utc)
{
out.captured_at = Some(dt);
}
@@ -410,6 +436,42 @@ fn read_nom_exif(path: &Path) -> NomExif {
out
}
/// Video arm: ONE open, dispatched on the sniffed container kind. Matches
/// the old `read_exif(path)`-then-`read_track(path)` observable behaviour
/// exactly — a Track container never parsed as EXIF (the old first open was
/// pure waste) and an Image container never parsed as a track, so the
/// two-open sequence always reduced to a single effective parse.
fn read_nom_exif_video(path: &Path) -> NomExif {
use nom_exif::{MediaKind, MediaParser, MediaSource, TrackInfoTag};
let mut out = NomExif::default();
let Ok(file) = std::fs::File::open(path) else {
return out;
};
let Ok(ms) = MediaSource::seekable(file) else {
return out;
};
let mut parser = MediaParser::new();
match ms.kind() {
MediaKind::Image => {
// MIME said video, bytes say image (mislabeled row): same EXIF
// extraction the old `read_exif(path)` performed.
if let Ok(iter) = parser.parse_exif(ms) {
let exif: nom_exif::Exif = iter.into();
nom_fill_from_exif(&exif, &mut out);
}
}
MediaKind::Track => {
if let Ok(track) = parser.parse_track(ms)
&& let Some(dt) = track.get(TrackInfoTag::CreateDate).and_then(nom_to_utc)
{
out.captured_at = Some(dt);
}
}
}
out
}
/// Combine kamadak's rich EXIF with nom-exif's date + GPS.
///
/// nom-exif's tz-correct date wins whenever present; its GPS only fills gaps
+3
View File
@@ -12,6 +12,7 @@ pub mod face_indexing_service;
pub mod ffmpeg_video_frame_service;
pub mod file_content_cache;
pub mod file_system_i18n_service;
pub mod grant_cleanup_service;
pub mod image_transcode_service;
pub mod jwt_service;
pub mod local_blob_backend;
@@ -33,6 +34,7 @@ pub mod path_service;
pub mod pg_acl_engine;
#[cfg(feature = "plugins")]
pub mod plugins;
pub mod recent_recording_hook;
pub mod retry_blob_backend;
pub mod s3_blob_backend;
pub mod search_index;
@@ -43,6 +45,7 @@ pub mod thumbnail_service;
mod thumbnail_service_test;
pub mod trash_cleanup_service;
pub mod tree_etag_flush_service;
pub mod webdav_dead_property_store;
pub mod webdav_lock_service;
pub mod wopi_discovery_service;
pub mod zip_service;
@@ -1,25 +1,84 @@
use std::path::PathBuf;
use std::time::Duration;
use tokio::fs;
use tokio::io::AsyncWriteExt;
use crate::common::errors::{DomainError, Result};
/// In-RAM running byte counter per upload session (`user/upload_id` →
/// bytes accepted so far). The per-chunk quota gate used to recompute
/// this by listing the whole session directory and stat-ing every chunk
/// on EVERY chunk PUT — O(k) stats for chunk k, O(N²/2) over an upload
/// (~500k stats for a 10 GB / 1000-chunk upload). The counter makes the
/// gate O(1); a cache miss (process restart, eviction) lazily rebuilds
/// from the directory listing, so crash-correctness is unchanged
/// (benches/NC-CHUNK-GATE.md). Sessions are forgotten on cleanup; the
/// TTL reaps counters for sessions the client abandoned.
fn build_session_bytes_cache() -> moka::sync::Cache<String, u64> {
moka::sync::Cache::builder()
.max_capacity(100_000)
.time_to_idle(Duration::from_secs(24 * 3600))
.build()
}
#[derive(Clone)]
pub struct NextcloudChunkedUploadService {
pub base_dir: PathBuf,
/// See [`build_session_bytes_cache`]. Cloning the service shares the
/// counter (moka `Cache` clones are handles to the same store).
session_bytes: moka::sync::Cache<String, u64>,
}
impl NextcloudChunkedUploadService {
pub fn new(base_dir: PathBuf) -> Self {
Self { base_dir }
Self {
base_dir,
session_bytes: build_session_bytes_cache(),
}
}
pub fn new_stub() -> Self {
Self {
base_dir: PathBuf::from("./storage/.uploads/nextcloud"),
session_bytes: build_session_bytes_cache(),
}
}
fn bytes_key(user: &str, upload_id: &str) -> String {
format!("{user}/{upload_id}")
}
/// Session bytes accepted so far, if the counter is warm.
/// `None` = rebuild from the directory listing and call
/// [`Self::set_session_bytes`].
pub fn cached_session_bytes(&self, user: &str, upload_id: &str) -> Option<u64> {
self.session_bytes.get(&Self::bytes_key(user, upload_id))
}
/// Seed / overwrite the session counter (post-rebuild or on MKCOL).
pub fn set_session_bytes(&self, user: &str, upload_id: &str, bytes: u64) {
self.session_bytes
.insert(Self::bytes_key(user, upload_id), bytes);
}
/// Add an accepted chunk's bytes to the counter (no-op when cold —
/// the next gate rebuilds from disk). Two racing PUTs on one session
/// could drop an increment; the counter is a gate hint, and the
/// MOVE-time quota check stays authoritative.
pub fn bump_session_bytes(&self, user: &str, upload_id: &str, delta: u64) {
let key = Self::bytes_key(user, upload_id);
if let Some(current) = self.session_bytes.get(&key) {
self.session_bytes
.insert(key, current.saturating_add(delta));
}
}
/// Drop the counter (session cleanup, or a chunk overwrite made the
/// running total untrustworthy — rebuilt lazily on next use).
pub fn forget_session_bytes(&self, user: &str, upload_id: &str) {
self.session_bytes
.invalidate(&Self::bytes_key(user, upload_id));
}
/// Validate that a path component contains no traversal characters.
fn validate_path_component(name: &str, label: &str) -> Result<()> {
if name.is_empty()
@@ -49,6 +108,7 @@ impl NextcloudChunkedUploadService {
fs::create_dir_all(&session_dir)
.await
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
self.set_session_bytes(user, upload_id, 0);
Ok(())
}
@@ -76,6 +136,20 @@ impl NextcloudChunkedUploadService {
/// `interfaces/upload_ingest::stream_body_to_path` helper to stream the
/// HTTP body directly to disk and avoid materialising the whole chunk
/// in RAM.
///
/// Uses `tokio::fs::write` (single `spawn_blocking` around
/// `std::fs::write`) rather than manually driving
/// `create + write_all` and letting the tokio handle drop close the
/// fd. The manual shape leaked a race: `tokio::fs::File::drop`
/// dispatches `close(2)` to the blocking pool without awaiting it,
/// and until close completes the dirent update may not be visible
/// to a subsequent `read_dir` — on macOS APFS routinely, on Linux
/// under I/O contention. In practice that turned into
/// `ordered_chunk_paths` silently missing a just-uploaded chunk;
/// the NC assembly path (`handle_assemble` → `ordered_chunk_paths`)
/// would then produce a truncated file with no error to the client.
/// `std::fs::write` opens, writes, and synchronously closes before
/// returning, so the dirent is guaranteed visible on `.await`.
pub async fn store_chunk(
&self,
user: &str,
@@ -84,12 +158,16 @@ impl NextcloudChunkedUploadService {
data: &[u8],
) -> Result<()> {
let chunk_path = self.safe_chunk_path(user, upload_id, chunk_name)?;
let mut file = fs::File::create(&chunk_path)
.await
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
file.write_all(data)
let overwrite = fs::metadata(&chunk_path).await.is_ok();
fs::write(&chunk_path, data)
.await
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
if overwrite {
// Retried chunk — running total is stale; rebuild lazily.
self.forget_session_bytes(user, upload_id);
} else {
self.bump_session_bytes(user, upload_id, data.len() as u64);
}
Ok(())
}
@@ -137,6 +215,7 @@ impl NextcloudChunkedUploadService {
.await
.map_err(|e| DomainError::internal_error("ChunkedUpload", e.to_string()))?;
}
self.forget_session_bytes(user, upload_id);
Ok(())
}
@@ -10,11 +10,13 @@ use std::sync::Arc;
use uuid::Uuid;
use crate::application::dtos::display_helpers::{
category_for, format_file_size, icon_class_for, icon_special_class_for,
classify_display, format_file_size, intern_display, intern_mime,
};
use crate::application::dtos::file_dto::FileDto;
use crate::application::dtos::folder_dto::FolderDto;
use crate::common::errors::DomainError;
use crate::domain::entities::file::File;
use crate::domain::entities::folder::Folder;
/// Result of resolving a WebDAV path — either a folder or a file.
#[derive(Debug, Clone)]
@@ -33,14 +35,20 @@ impl PathResolverService {
Self { pool }
}
/// Resolve `path` to a folder or file **owned by `user_id`**.
/// Resolve `path` to a folder or file **within the given drive**.
///
/// Adds `AND fo.user_id = $4` / `AND fi.user_id = $4` so that one
/// user can never resolve another user's resources.
pub async fn resolve_path_for_user(
/// Filters on `fo.drive_id = $4` / `fi.drive_id = $4`. Callers
/// pre-resolve which drive they're operating in — native WebDAV
/// derives it from the caller's default drive
/// (`resolve_drive_id_for_native_webdav`); NC WebDAV takes it from
/// the URL-selected chroot (`chroot.drive_id`). Shared by both
/// surfaces so the single-query UNION ALL optimisation lands
/// consistently and no path lookup keys on the doomed
/// `storage.{files,folders}.user_id` column.
pub async fn resolve_path_in_drive(
&self,
path: &str,
user_id: Uuid,
drive_id: Uuid,
) -> Result<ResolvedResource, DomainError> {
let path = path.trim_start_matches('/').trim_end_matches('/');
if path.is_empty() {
@@ -55,6 +63,13 @@ impl PathResolverService {
String::new()
};
// Widened SELECT: also fetches `blob_hash` (for file ETag) and
// `tree_modified_at` (for folder ETag). Both share the same
// canonical formulas as the rest of the codebase — see
// [`File::compute_etag`] and [`Folder::compute_etag`]. Without
// these two extra columns the resolver used to emit empty
// ETag strings, and NC's `If-Match` round-trips broke
// (see the F6b regression on `test_nc_put_mkcol_blake3.sh`).
let row = sqlx::query_as::<
_,
(
@@ -63,34 +78,37 @@ impl PathResolverService {
String, // name
String, // path
Option<String>, // parent_id
Option<String>, // user_id
Uuid, // drive_id
i64, // created_at
i64, // modified_at
Option<i64>, // size
Option<String>, // mime_type
Option<String>, // folder_id
Option<String>, // blob_hash (files only)
Option<i64>, // tree_modified_at (folders only)
),
>(
r#"
SELECT resource_type, id, name, path, parent_id, user_id, drive_id,
created_at, modified_at, size, mime_type, folder_id
SELECT resource_type, id, name, path, parent_id, drive_id,
created_at, modified_at, size, mime_type, folder_id,
blob_hash, tree_modified_at
FROM (
SELECT 'folder'::text AS resource_type,
fo.id::text,
fo.name,
fo.path,
fo.parent_id::text,
fo.user_id::text,
fo.drive_id,
EXTRACT(EPOCH FROM fo.created_at)::bigint AS created_at,
EXTRACT(EPOCH FROM fo.updated_at)::bigint AS modified_at,
NULL::bigint AS size,
NULL::text AS mime_type,
NULL::text AS folder_id
NULL::text AS folder_id,
NULL::text AS blob_hash,
EXTRACT(EPOCH FROM fo.tree_modified_at)::bigint AS tree_modified_at
FROM storage.folders fo
WHERE fo.path = $1 AND NOT fo.is_trashed
AND fo.user_id = $4
AND fo.drive_id = $4
UNION ALL
@@ -103,13 +121,14 @@ impl PathResolverService {
ELSE fi.name
END AS path,
NULL::text AS parent_id,
fi.user_id::text,
fi.drive_id,
EXTRACT(EPOCH FROM fi.created_at)::bigint AS created_at,
EXTRACT(EPOCH FROM fi.updated_at)::bigint AS modified_at,
fi.size,
fi.mime_type,
fi.folder_id::text
fi.folder_id::text,
fi.blob_hash,
NULL::bigint AS tree_modified_at
FROM storage.files fi
LEFT JOIN storage.folders fo ON fo.id = fi.folder_id
WHERE fi.name = $2
@@ -118,7 +137,7 @@ impl PathResolverService {
OR fo.path = $3
)
AND NOT fi.is_trashed
AND fi.user_id = $4
AND fi.drive_id = $4
) sub
LIMIT 1
"#,
@@ -126,10 +145,10 @@ impl PathResolverService {
.bind(path) // $1
.bind(filename) // $2
.bind(&folder_path) // $3
.bind(user_id) // $4
.bind(drive_id) // $4
.fetch_optional(self.pool.as_ref())
.await
.map_err(|e| DomainError::internal_error("PathResolver", format!("resolve_for_user: {e}")))?
.map_err(|e| DomainError::internal_error("PathResolver", format!("resolve_in_drive: {e}")))?
.ok_or_else(|| DomainError::not_found("Resource", path))?;
let (
@@ -138,62 +157,63 @@ impl PathResolverService {
name,
res_path,
parent_id,
uid,
drive_id,
created_at,
modified_at,
size,
mime_type,
folder_id,
blob_hash,
tree_modified_at,
) = row;
match resource_type.as_str() {
"folder" => Ok(ResolvedResource::Folder(FolderDto {
etag: id.clone(),
id,
name: name.clone(),
path: res_path,
parent_id,
owner_id: uid,
drive_id,
created_at: created_at as u64,
modified_at: modified_at as u64,
is_root: false,
icon_class: Arc::from("fas fa-folder"),
icon_special_class: Arc::from("folder-icon"),
category: Arc::from("Folder"),
// §14 provenance not selected by this resolver path —
// it's used for existence/type discrimination, not
// detailed DTO emission. Callers that need provenance
// reload through the repo.
created_by: None,
updated_by: None,
})),
"folder" => {
let tree_mod = tree_modified_at.unwrap_or(modified_at) as u64;
Ok(ResolvedResource::Folder(FolderDto {
etag: Folder::compute_etag(&id, tree_mod),
id,
name: name.clone(),
path: res_path,
parent_id,
drive_id,
created_at: created_at as u64,
modified_at: modified_at as u64,
is_root: false,
icon_class: intern_display("fas fa-folder"),
icon_special_class: intern_display("folder-icon"),
category: intern_display("Folder"),
// §14 provenance not selected by this resolver path —
// it's used for existence/type discrimination, not
// detailed DTO emission. Callers that need provenance
// reload through the repo.
created_by: None,
updated_by: None,
}))
}
_ => {
let mime = mime_type.unwrap_or_else(|| "application/octet-stream".to_string());
let sz = size.unwrap_or(0) as u64;
// `content_hash`/`etag` are empty here: this resolver
// path doesn't select `blob_hash` from SQL — callers
// are doing existence/type discrimination, not ETag
// emission. If a caller ever needs an ETag from this
// codepath, widen the SELECT and populate properly.
let hash = blob_hash.unwrap_or_default();
let modified_at_u = modified_at as u64;
let etag = File::compute_etag(&hash, modified_at_u);
let classes = classify_display(&name, &mime);
Ok(ResolvedResource::File(FileDto {
id,
name: name.clone(),
path: res_path,
size: sz,
mime_type: Arc::from(&*mime),
mime_type: intern_mime(&mime),
folder_id,
created_at: created_at as u64,
modified_at: modified_at as u64,
icon_class: Arc::from(icon_class_for(&name, &mime)),
icon_special_class: Arc::from(icon_special_class_for(&name, &mime)),
category: Arc::from(category_for(&name, &mime)),
modified_at: modified_at_u,
icon_class: intern_display(classes.icon_class),
icon_special_class: intern_display(classes.icon_special_class),
category: intern_display(classes.category),
size_formatted: format_file_size(sz),
owner_id: uid,
sort_date: None,
content_hash: String::new(),
etag: String::new(),
content_hash: hash,
etag,
// §14 provenance not selected by this resolver path
created_by: None,
updated_by: None,
@@ -203,7 +223,10 @@ impl PathResolverService {
}
/// Returns `true` if the resource at `path` belongs to `user_id`.
pub async fn exists_for_user(&self, path: &str, user_id: Uuid) -> Result<bool, DomainError> {
/// Check whether `path` resolves to a folder or file within the
/// given drive. Companion to `resolve_path_in_drive` — same scope
/// filter, existence-only projection.
pub async fn exists_in_drive(&self, path: &str, drive_id: Uuid) -> Result<bool, DomainError> {
let path = path.trim_start_matches('/').trim_end_matches('/');
if path.is_empty() {
return Ok(false);
@@ -221,7 +244,7 @@ impl PathResolverService {
r#"
SELECT EXISTS(
SELECT 1 FROM storage.folders
WHERE path = $1 AND NOT is_trashed AND user_id = $4
WHERE path = $1 AND NOT is_trashed AND drive_id = $4
) OR EXISTS(
SELECT 1
FROM storage.files fi
@@ -229,18 +252,18 @@ impl PathResolverService {
WHERE fi.name = $2
AND (($3 = '' AND fi.folder_id IS NULL) OR fo.path = $3)
AND NOT fi.is_trashed
AND fi.user_id = $4
AND fi.drive_id = $4
)
"#,
)
.bind(path)
.bind(filename)
.bind(&folder_path)
.bind(user_id)
.bind(drive_id)
.fetch_one(self.pool.as_ref())
.await
.map_err(|e| {
DomainError::internal_error("PathResolver", format!("exists_for_user: {e}"))
DomainError::internal_error("PathResolver", format!("exists_in_drive: {e}"))
})?;
Ok(exists)
+1 -1
View File
@@ -95,7 +95,7 @@ impl PathService {
/// Validates a path to ensure it doesn't contain dangerous components
pub fn validate_path(&self, path: &StoragePath) -> Result<(), DomainError> {
// Check for empty segments
if path.segments().iter().any(|s| s.is_empty()) {
if path.segments().any(|s| s.is_empty()) {
return Err(DomainError::new(
ErrorKind::InvalidInput,
"Path",
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,120 @@
//! Recording side of [`ResourceAccessHook`] — turns a successful file access
//! into a row in `auth.user_recent_files` via [`RecentService`].
//!
//! Wiring lives in `common/di.rs`: this hook is registered once, every
//! `_with_perms` file method on `FileRetrievalService` / `FileManagementService`
//! fans through it, and any future read-path or write-path service can opt in
//! by holding an `Option<Arc<dyn ResourceAccessHook>>` and calling
//! `on_file_accessed` after authZ.
//!
//! Two non-obvious behaviours, with rationale:
//!
//! * **Per-(caller, file) 60-second throttle.** Range-stream downloads send
//! one GET per chunk (NC desktop, video seek, resumable transfers); without
//! throttling each chunk would trigger an upsert against the same row.
//! Moka's `time_to_live` gives us bounded memory and lock-free reads. The
//! underlying `INSERT … ON CONFLICT DO UPDATE accessed_at = now()` is
//! idempotent, so the rare TOCTOU window between `contains_key` and `insert`
//! is harmless — at worst we record twice for the same instant.
//!
//! * **Fire-and-forget via `tokio::spawn`.** The `ResourceAccessHook` method
//! is synchronous by contract (every `with_perms` caller would otherwise
//! have to `await` the side-effect). The spawn lets the user-facing
//! response return immediately; a DB hiccup in Recent recording never
//! bubbles up to the GET / PUT that triggered it. Failures log at warn.
use std::sync::Arc;
use std::time::Duration;
use moka::sync::Cache;
use uuid::Uuid;
use crate::application::ports::resource_access_hook::ResourceAccessHook;
use crate::application::services::recent_service::RecentService;
/// How long a successful recording suppresses repeat upserts for the same
/// `(caller, file)`. Sized to span a typical streamed range-GET burst while
/// still updating `accessed_at` often enough that the Recent list reflects
/// "this is the file I was just looking at".
const THROTTLE_TTL_SECONDS: u64 = 60;
/// Bound on simultaneous in-flight throttle entries. Each entry is a tuple
/// `(Uuid, String) -> ()` ≈ 80 B; 16 384 entries ≈ 1.3 MB worst case. LRU
/// eviction keeps memory bounded even if a pathological client touches a
/// million files in a minute.
const THROTTLE_MAX_ENTRIES: u64 = 16_384;
/// `ResourceAccessHook` implementation that records file accesses into
/// `auth.user_recent_files`, throttled per (caller, file).
pub struct RecentRecordingHook {
recent: Arc<RecentService>,
throttle: Cache<(Uuid, String), ()>,
}
impl RecentRecordingHook {
pub fn new(recent: Arc<RecentService>) -> Self {
// `support_invalidation_closures` is the moka opt-in needed by
// `invalidate_entries_if` (the per-user throttle reset on
// `on_recents_cleared`). Without it, the predicate-based
// invalidate call silently no-ops and a freshly-cleared Recent
// list refuses to re-record the same file until the TTL
// expires — exactly the bug surfaced by tests/api/recent.hurl
// step 8.
let throttle = Cache::builder()
.max_capacity(THROTTLE_MAX_ENTRIES)
.time_to_live(Duration::from_secs(THROTTLE_TTL_SECONDS))
.support_invalidation_closures()
.build();
Self { recent, throttle }
}
}
impl ResourceAccessHook for RecentRecordingHook {
fn on_file_accessed(&self, caller_id: Uuid, file_id: &str) {
let key = (caller_id, file_id.to_string());
if self.throttle.contains_key(&key) {
return;
}
// Insert before spawning: even if the spawned task races with another
// call for the same key, the cache entry suppresses the duplicate
// before it reaches the DB. The ON CONFLICT clause covers the
// sub-microsecond TOCTOU window between contains_key and insert.
self.throttle.insert(key.clone(), ());
let recent = Arc::clone(&self.recent);
let (caller_id, file_id) = key;
tokio::spawn(async move {
// Fast path: skip the trait's `authz.require(Read, …)`
// (upstream `_with_perms` service already gated). The
// extra SQL round-trip pushes the upsert past the client's
// immediate `GET /api/recent/resources` in
// `tests/api/recent.hurl` step 7 — the whole reason for
// the internal variant.
if let Err(e) = recent
.record_item_access_internal(caller_id, &file_id, "file")
.await
{
tracing::warn!(
target: "oxicloud::recent",
caller_id = %caller_id,
file_id = %file_id,
"recent recording failed: {e}",
);
}
});
}
fn on_recents_cleared(&self, caller_id: Uuid) {
// Drop every throttle entry that would otherwise suppress the
// next recording for this user. moka schedules the predicate to
// run during the next maintenance pass — it's not synchronous.
// The DB clear has already happened by the time we get here, so
// any racing access between the clear and the next maintenance
// pass just re-records via ON CONFLICT — the worst case is a row
// that surfaces in Recent a few ms after the clear, which is
// exactly what the user asked for.
let _ = self
.throttle
.invalidate_entries_if(move |(k_caller, _), _| *k_caller == caller_id);
}
}
+127 -48
View File
@@ -57,14 +57,20 @@ impl RetryBlobBackend {
}
/// Execute an async closure with exponential backoff retry.
async fn retry_async<F, Fut, T>(
///
/// `name` is a lazy label: the success path (the overwhelmingly common
/// case) never materializes it, so per-op `format!("op({hash})")`
/// allocations only happen on an actual retry (benches/ROUND11.md §14:
/// 64.5 → 0.7 ns, −2 allocs per blob op).
async fn retry_async<F, Fut, T, L>(
policy: &RetryPolicy,
name: &str,
name: L,
mut f: F,
) -> Result<T, DomainError>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T, DomainError>>,
L: Fn() -> String,
{
let mut attempt = 0u32;
let mut backoff = policy.initial_backoff;
@@ -78,7 +84,7 @@ where
"Retry {}/{} for {} after error: {} (backoff {:?})",
attempt,
policy.max_retries,
name,
name(),
e,
backoff
);
@@ -112,10 +118,14 @@ impl BlobStorageBackend for RetryBlobBackend {
let inner = self.inner.clone();
let policy = self.policy.clone();
Box::pin(async move {
retry_async(&policy, "initialize", || {
let inner = inner.clone();
async move { inner.initialize().await }
})
retry_async(
&policy,
|| "initialize".to_string(),
|| {
let inner = inner.clone();
async move { inner.initialize().await }
},
)
.await
})
}
@@ -130,12 +140,16 @@ impl BlobStorageBackend for RetryBlobBackend {
let hash = hash.to_string();
let path = source_path.to_path_buf();
Box::pin(async move {
retry_async(&policy, &format!("put_blob({hash})"), || {
let inner = inner.clone();
let hash = hash.clone();
let path = path.clone();
async move { inner.put_blob(&hash, &path).await }
})
retry_async(
&policy,
|| format!("put_blob({hash})"),
|| {
let inner = inner.clone();
let hash = hash.clone();
let path = path.clone();
async move { inner.put_blob(&hash, &path).await }
},
)
.await
})
}
@@ -149,16 +163,57 @@ impl BlobStorageBackend for RetryBlobBackend {
let policy = self.policy.clone();
let hash = hash.to_string();
Box::pin(async move {
retry_async(&policy, &format!("put_blob_from_bytes({hash})"), || {
let inner = inner.clone();
let hash = hash.clone();
let data = data.clone();
async move { inner.put_blob_from_bytes(&hash, data).await }
})
retry_async(
&policy,
|| format!("put_blob_from_bytes({hash})"),
|| {
let inner = inner.clone();
let hash = hash.clone();
let data = data.clone();
async move { inner.put_blob_from_bytes(&hash, data).await }
},
)
.await
})
}
// Without this override the trait default would re-route the CDC chunk
// write through `put_blob_from_bytes` above — reinstating the remote
// backend's exists-probe (HEAD/get_properties) per chunk that the
// `_unsynced` fast path exists to skip.
fn put_blob_from_bytes_unsynced(
&self,
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let inner = self.inner.clone();
let policy = self.policy.clone();
let hash = hash.to_string();
Box::pin(async move {
retry_async(
&policy,
|| format!("put_blob_from_bytes_unsynced({hash})"),
|| {
let inner = inner.clone();
let hash = hash.clone();
let data = data.clone();
async move { inner.put_blob_from_bytes_unsynced(&hash, data).await }
},
)
.await
})
}
// Forwarded WITHOUT retry wrapping: a failed fsync must surface, not be
// re-issued — after an fsync error the kernel may have dropped the dirty
// pages, so a retried fsync can report success for data that was lost.
fn sync_blobs(
&self,
hashes: &[String],
) -> Pin<Box<dyn std::future::Future<Output = Result<(), DomainError>> + Send + '_>> {
self.inner.sync_blobs(hashes)
}
fn get_blob_stream(
&self,
hash: &str,
@@ -168,11 +223,15 @@ impl BlobStorageBackend for RetryBlobBackend {
let policy = self.policy.clone();
let hash = hash.to_string();
Box::pin(async move {
retry_async(&policy, &format!("get_blob_stream({hash})"), || {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.get_blob_stream(&hash).await }
})
retry_async(
&policy,
|| format!("get_blob_stream({hash})"),
|| {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.get_blob_stream(&hash).await }
},
)
.await
})
}
@@ -188,11 +247,15 @@ impl BlobStorageBackend for RetryBlobBackend {
let policy = self.policy.clone();
let hash = hash.to_string();
Box::pin(async move {
retry_async(&policy, &format!("get_blob_range({hash})"), || {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.get_blob_range_stream(&hash, start, end).await }
})
retry_async(
&policy,
|| format!("get_blob_range({hash})"),
|| {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.get_blob_range_stream(&hash, start, end).await }
},
)
.await
})
}
@@ -205,11 +268,15 @@ impl BlobStorageBackend for RetryBlobBackend {
let policy = self.policy.clone();
let hash = hash.to_string();
Box::pin(async move {
retry_async(&policy, &format!("delete_blob({hash})"), || {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.delete_blob(&hash).await }
})
retry_async(
&policy,
|| format!("delete_blob({hash})"),
|| {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.delete_blob(&hash).await }
},
)
.await
})
}
@@ -222,11 +289,15 @@ impl BlobStorageBackend for RetryBlobBackend {
let policy = self.policy.clone();
let hash = hash.to_string();
Box::pin(async move {
retry_async(&policy, &format!("blob_exists({hash})"), || {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.blob_exists(&hash).await }
})
retry_async(
&policy,
|| format!("blob_exists({hash})"),
|| {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.blob_exists(&hash).await }
},
)
.await
})
}
@@ -239,11 +310,15 @@ impl BlobStorageBackend for RetryBlobBackend {
let policy = self.policy.clone();
let hash = hash.to_string();
Box::pin(async move {
retry_async(&policy, &format!("blob_size({hash})"), || {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.blob_size(&hash).await }
})
retry_async(
&policy,
|| format!("blob_size({hash})"),
|| {
let inner = inner.clone();
let hash = hash.clone();
async move { inner.blob_size(&hash).await }
},
)
.await
})
}
@@ -256,10 +331,14 @@ impl BlobStorageBackend for RetryBlobBackend {
let inner = self.inner.clone();
let policy = self.policy.clone();
Box::pin(async move {
retry_async(&policy, "health_check", || {
let inner = inner.clone();
async move { inner.health_check().await }
})
retry_async(
&policy,
|| "health_check".to_string(),
|| {
let inner = inner.clone();
async move { inner.health_check().await }
},
)
.await
})
}
@@ -200,6 +200,38 @@ impl BlobStorageBackend for S3BlobBackend {
})
}
/// Dedup settle path: PUT unconditionally. Keys are content-addressed
/// (BLAKE3), so a re-PUT writes identical bytes — overwrite-safe
/// idempotency without the HEAD probe `put_blob_from_bytes` pays. The
/// dedup layer already filtered out chunks the database knows about,
/// so the probe was a pure extra round-trip on every NEW chunk of
/// every upload (2 RTTs -> 1, benches/S3-PUT.md).
fn put_blob_from_bytes_unsynced(
&self,
hash: &str,
data: Bytes,
) -> Pin<Box<dyn std::future::Future<Output = Result<u64, DomainError>> + Send + '_>> {
let hash = hash.to_owned();
Box::pin(async move {
let key = Self::object_key(&hash);
let size = data.len() as u64;
self.client
.put_object()
.bucket(&self.bucket)
.key(&key)
.body(ByteStream::from(data))
.send()
.await
.map_err(|e| {
DomainError::internal_error(
"S3",
format!("Failed to upload blob {}: {}", hash, e),
)
})?;
Ok(size)
})
}
fn get_blob_stream(
&self,
hash: &str,
@@ -242,30 +242,43 @@ impl ContentIndexWorker {
// Authoritative state re-read: a queued 'upsert' whose row vanished
// or got trashed in the meantime becomes a delete.
let files: Vec<(Uuid, String, String, String, String, String, i64)> =
if upsert_candidates.is_empty() {
Vec::new()
} else {
sqlx::query_as(
"SELECT fi.id, fi.user_id::text, fi.drive_id::text, fi.name,
fi.blob_hash, fi.mime_type, fi.size
FROM storage.files fi
WHERE fi.id = ANY($1) AND NOT fi.is_trashed",
)
.bind(&upsert_candidates)
.fetch_all(self.maintenance_pool.as_ref())
.await?
};
//
// Post-D7: `fi.user_id` is dropped — no longer projected. The
// Tantivy `user_id` field survives as defence-in-depth but now
// always indexes `""`. Every query is Must-scoped by `drive_id`.
// (file_id, drive_id, name, blob_hash, mime, size).
type FileIndexRow = (Uuid, String, String, String, String, i64);
let files: Vec<FileIndexRow> = if upsert_candidates.is_empty() {
Vec::new()
} else {
sqlx::query_as(
"SELECT fi.id, fi.drive_id::text, fi.name,
fi.blob_hash, fi.mime_type, fi.size
FROM storage.files fi
WHERE fi.id = ANY($1) AND NOT fi.is_trashed",
)
.bind(&upsert_candidates)
.fetch_all(self.maintenance_pool.as_ref())
.await?
};
let found: HashSet<Uuid> = files.iter().map(|f| f.0).collect();
deletes.extend(upsert_candidates.iter().filter(|id| !found.contains(id)));
// `supports` lowercases the MIME (and, on a generic MIME, the extension)
// — 1–2 allocations per call. Classify each file ONCE here and thread the
// flag through both the wanted-hashes filter and the per-file records
// loop below, where it used to be re-derived a second time per file.
let supported: Vec<bool> = files
.iter()
.map(|(_, _, name, _, mime, _)| text_extractor::supports(name, mime))
.collect();
// Per-blob text: batch-read the extraction cache, extract misses.
let wanted_hashes: Vec<String> = files
.iter()
.filter(|(_, _, _, name, _, mime, size)| {
text_extractor::supports(name, mime) && *size as u64 <= self.max_extract_file_bytes
})
.map(|f| f.4.clone())
.zip(&supported)
.filter(|&(f, sup)| *sup && f.5 as u64 <= self.max_extract_file_bytes)
.map(|(f, _)| f.3.clone())
.collect();
let mut text_by_hash: HashMap<String, Option<String>> = HashMap::new();
if !wanted_hashes.is_empty() {
@@ -282,8 +295,9 @@ impl ContentIndexWorker {
}
let mut records = Vec::with_capacity(files.len());
for (file_id, user_id, drive_id, name, blob_hash, mime, size) in files {
let supported = text_extractor::supports(&name, &mime);
for ((file_id, drive_id, name, blob_hash, mime, size), supported) in
files.into_iter().zip(supported)
{
let content = if !supported {
None
} else if let Some(cached) = text_by_hash.get(&blob_hash) {
@@ -301,7 +315,7 @@ impl ContentIndexWorker {
.map(|t| truncate_on_char(t, PREVIEW_BYTES));
records.push(IndexDocRecord {
file_id: file_id.to_string(),
user_id,
user_id: String::new(),
drive_id,
name,
content,
@@ -238,8 +238,10 @@ impl TantivyContentIndex {
}
/// Tokenize `raw` with the index analyzer (simple split + lowercase).
fn query_tokens(analyzer: &TextAnalyzer, raw: &str) -> Vec<String> {
let mut analyzer = analyzer.clone();
/// Takes the analyzer by value — the caller's per-search clone is the
/// only one needed; cloning the boxed tokenizer chain again here doubled
/// the per-query allocation for nothing.
fn query_tokens(mut analyzer: TextAnalyzer, raw: &str) -> Vec<String> {
let mut tokens = Vec::new();
let mut stream = analyzer.token_stream(raw);
while stream.advance() && tokens.len() < MAX_QUERY_TOKENS {
@@ -337,7 +339,7 @@ impl TantivyContentIndex {
raw_query: &str,
limit: usize,
) -> Result<Vec<ContentHitDto>, DomainError> {
let tokens = Self::query_tokens(&analyzer, raw_query);
let tokens = Self::query_tokens(analyzer, raw_query);
if tokens.is_empty() {
return Ok(Vec::new());
}
@@ -347,6 +349,14 @@ impl TantivyContentIndex {
.search(&query, &TopDocs::with_limit(limit.max(1)).order_by_score())
.map_err(|e| DomainError::internal_error("ContentIndex", format!("search: {e}")))?;
// No hits → no documents to highlight. `SnippetGenerator::create`
// compiles the query against the index (term lookups + weight build);
// for a query that matched nothing that is pure waste on the search
// request path, and the per-hit loop below never runs. Return early.
if top_docs.is_empty() {
return Ok(Vec::new());
}
// Snippets highlight CONTENT matches; an empty fragment means the hit
// came from the name (or a fuzzy variant) — no snippet then.
let snippet_generator = SnippetGenerator::create(&searcher, &*query, fields.content)
@@ -243,7 +243,10 @@ fn collect_xml_text<R: std::io::BufRead>(
}
match xml.read_event_into(&mut buf) {
Ok(Event::Text(t)) => {
if let Ok(decoded) = t.xml_content() {
// quick-xml 0.41+ makes XmlVersion explicit on xml_content()
// so callers pick 1.0 vs 1.1 entity-normalization rules. Text
// extraction is version-agnostic — 1.0 is the sane default.
if let Ok(decoded) = t.xml_content(quick_xml::XmlVersion::Implicit1_0) {
out.push_str(&decoded);
}
}
@@ -1479,7 +1479,7 @@ impl crate::application::ports::blob_lifecycle::BlobLifecycleHook for ThumbnailS
for format in [ThumbnailFormat::Webp, ThumbnailFormat::Jpeg] {
let path =
root.join(size.dir_name())
.join(format!("{}.{}", &blob_hash, format.ext()));
.join(format!("{}.{}", blob_hash, format.ext()));
if tokio::fs::metadata(&path).await.is_ok() {
let _ = tokio::fs::remove_file(&path).await;
}
@@ -0,0 +1,264 @@
//! PostgreSQL-backed dead property store for WebDAV PROPPATCH / PROPFIND compliance.
//!
//! RFC 4918 §4.2 defines "dead properties" as those stored verbatim by the
//! server without interpreting their value. Properties are persisted to
//! `storage.webdav_dead_properties` and survive server restarts.
//!
//! Keying contract (after migration 20260830000001): the row is keyed by
//! the underlying resource id — exactly one of `folder_id` / `file_id` is
//! set — not by the resource's current path. Three consequences:
//!
//! * Every delete code path (REST, WebDAV, NextCloud DAV, trash empty,
//! folder cascade) reaps dead-property rows for free via FK
//! `ON DELETE CASCADE`. The store has no `remove_resource()` method
//! because it isn't needed: deleting the file/folder row reaps the
//! attached dead properties as a database invariant.
//! * MOVE / RENAME never changes the resource id, so dead properties
//! follow the resource without any store-side bookkeeping. The store
//! has no `rename_resource()` method for the same reason.
//! * Dead properties are RESOURCE state (RFC 4918 §4.2), not user
//! state. Two users on a shared drive PROPFIND'ing the same resource
//! see the same dead properties. The `user_id` scope key from the
//! pre-rekey schema is gone; user-delete cleanup happens
//! transitively through `auth.users` → `storage.{folders,files}` →
//! this table.
//!
//! Queries use `sqlx::query()` (runtime-bound) rather than `sqlx::query!()`
//! to keep fresh checkouts compilable without a DB connection — the
//! codebase's standing convention.
//!
//! COPY semantics (RFC 4918 §8.8 — dead properties MUST be duplicated)
//! are NOT handled here. The COPY handler is responsible for explicitly
//! reading the source's dead properties via `get_all()` and writing them
//! against the new resource id via `set()`. Not done in this migration —
//! it was not handled by the path-based store either, so this is a
//! parity decision, not a regression.
use std::collections::HashMap;
use std::sync::Arc;
use sqlx::{PgPool, Row};
use uuid::Uuid;
use crate::application::adapters::webdav_adapter::QualifiedName;
use crate::domain::errors::DomainError;
/// Polymorphic reference to the resource a dead property hangs off.
///
/// Exactly one variant — folder or file — is ever stored in a single
/// row. The CHECK constraint
/// `(folder_id IS NULL) <> (file_id IS NULL)` enforces this at the
/// database level so the application layer cannot accidentally write a
/// row that's both or neither.
#[derive(Clone, Copy, Debug)]
pub enum ResourceRef {
Folder(Uuid),
File(Uuid),
}
pub struct DeadPropertyStore {
pool: Arc<PgPool>,
}
impl DeadPropertyStore {
pub fn new(pool: Arc<PgPool>) -> Self {
Self { pool }
}
/// Upsert a dead property. `value = None` means an empty XML element.
///
/// The two SQL branches are deliberately kept separate so each
/// ON CONFLICT clause can target the matching partial unique
/// index (`idx_webdav_dead_props_folder_unique` /
/// `idx_webdav_dead_props_file_unique`). A combined upsert would
/// require a non-partial unique index that treats NULL as
/// distinct, which doesn't match the (folder XOR file) shape.
pub async fn set(
&self,
r: ResourceRef,
name: QualifiedName,
value: Option<String>,
) -> Result<(), DomainError> {
match r {
ResourceRef::Folder(folder_id) => {
sqlx::query(
r#"
INSERT INTO storage.webdav_dead_properties
(folder_id, namespace, local_name, value)
VALUES ($1, $2, $3, $4)
ON CONFLICT (folder_id, namespace, local_name)
WHERE folder_id IS NOT NULL
DO UPDATE SET value = EXCLUDED.value, updated_at = CURRENT_TIMESTAMP
"#,
)
.bind(folder_id)
.bind(&name.namespace)
.bind(&name.name)
.bind(&value)
.execute(&*self.pool)
.await
.map_err(|e| {
DomainError::internal_error("DeadPropertyStore", format!("set folder: {e}"))
})?;
}
ResourceRef::File(file_id) => {
sqlx::query(
r#"
INSERT INTO storage.webdav_dead_properties
(file_id, namespace, local_name, value)
VALUES ($1, $2, $3, $4)
ON CONFLICT (file_id, namespace, local_name)
WHERE file_id IS NOT NULL
DO UPDATE SET value = EXCLUDED.value, updated_at = CURRENT_TIMESTAMP
"#,
)
.bind(file_id)
.bind(&name.namespace)
.bind(&name.name)
.bind(&value)
.execute(&*self.pool)
.await
.map_err(|e| {
DomainError::internal_error("DeadPropertyStore", format!("set file: {e}"))
})?;
}
}
Ok(())
}
/// Delete a specific dead property. No-op if not present.
///
/// Filters on the concrete id column (`folder_id = $1` / `file_id = $1`)
/// rather than the old `IS NOT DISTINCT FROM` pair — PostgreSQL cannot
/// serve `IS NOT DISTINCT FROM` from a B-tree index, so every lookup
/// degraded to a sequential scan as the table grew. The `=` shape is
/// served by the partial unique indexes from migration 20260830000001.
/// (Same rationale for `get_all` / `get` / the batched readers below —
/// measured in `benches/DEAD-PROPS.md`.)
pub async fn remove(&self, r: ResourceRef, name: &QualifiedName) -> Result<(), DomainError> {
let (column, id) = split_ref(r);
sqlx::query(&format!(
"DELETE FROM storage.webdav_dead_properties
WHERE {column} = $1
AND namespace = $2
AND local_name = $3",
))
.bind(id)
.bind(&name.namespace)
.bind(&name.name)
.execute(&*self.pool)
.await
.map_err(|e| DomainError::internal_error("DeadPropertyStore", format!("remove: {e}")))?;
Ok(())
}
/// Return all dead properties for the given resource.
pub async fn get_all(
&self,
r: ResourceRef,
) -> Result<Vec<(QualifiedName, Option<String>)>, DomainError> {
let (column, id) = split_ref(r);
let rows = sqlx::query(&format!(
"SELECT namespace, local_name, value
FROM storage.webdav_dead_properties
WHERE {column} = $1",
))
.bind(id)
.fetch_all(&*self.pool)
.await
.map_err(|e| DomainError::internal_error("DeadPropertyStore", format!("get_all: {e}")))?;
Ok(rows.into_iter().map(row_to_prop).collect())
}
/// Batched variant of [`get_all`] for every file in a PROPFIND page:
/// ONE `file_id = ANY($1)` round-trip instead of N sequential queries.
/// Files with no dead properties are simply absent from the map.
pub async fn get_all_for_files(
&self,
file_ids: &[Uuid],
) -> Result<HashMap<Uuid, Vec<(QualifiedName, Option<String>)>>, DomainError> {
self.get_all_batched("file_id", file_ids).await
}
/// Batched variant of [`get_all`] for every subfolder in a PROPFIND page.
pub async fn get_all_for_folders(
&self,
folder_ids: &[Uuid],
) -> Result<HashMap<Uuid, Vec<(QualifiedName, Option<String>)>>, DomainError> {
self.get_all_batched("folder_id", folder_ids).await
}
async fn get_all_batched(
&self,
column: &str,
ids: &[Uuid],
) -> Result<HashMap<Uuid, Vec<(QualifiedName, Option<String>)>>, DomainError> {
if ids.is_empty() {
return Ok(HashMap::new());
}
let rows = sqlx::query(&format!(
"SELECT {column} AS resource_id, namespace, local_name, value
FROM storage.webdav_dead_properties
WHERE {column} = ANY($1)",
))
.bind(ids)
.fetch_all(&*self.pool)
.await
.map_err(|e| {
DomainError::internal_error("DeadPropertyStore", format!("get_all_batched: {e}"))
})?;
let mut map: HashMap<Uuid, Vec<(QualifiedName, Option<String>)>> = HashMap::new();
for row in rows {
let resource_id: Uuid = row.get("resource_id");
map.entry(resource_id).or_default().push(row_to_prop(row));
}
Ok(map)
}
/// Return a specific dead property, or `None` if not stored.
/// Returns `Some(None)` when the property exists with an empty value.
pub async fn get(
&self,
r: ResourceRef,
name: &QualifiedName,
) -> Result<Option<Option<String>>, DomainError> {
let (column, id) = split_ref(r);
let row = sqlx::query(&format!(
"SELECT value FROM storage.webdav_dead_properties
WHERE {column} = $1
AND namespace = $2
AND local_name = $3",
))
.bind(id)
.bind(&name.namespace)
.bind(&name.name)
.fetch_optional(&*self.pool)
.await
.map_err(|e| DomainError::internal_error("DeadPropertyStore", format!("get: {e}")))?;
Ok(row.map(|r| r.get::<Option<String>, _>("value")))
}
}
/// Maps a `ResourceRef` onto the column that stores it plus the id to bind.
/// The column name is one of two compile-time literals — never user input —
/// so interpolating it into the SQL text is safe.
fn split_ref(r: ResourceRef) -> (&'static str, Uuid) {
match r {
ResourceRef::Folder(id) => ("folder_id", id),
ResourceRef::File(id) => ("file_id", id),
}
}
fn row_to_prop(r: sqlx::postgres::PgRow) -> (QualifiedName, Option<String>) {
let namespace: String = r.get("namespace");
let local_name: String = r.get("local_name");
let value: Option<String> = r.get("value");
(QualifiedName::new(namespace, local_name), value)
}
pub fn create_dead_property_store(pool: Arc<PgPool>) -> Arc<DeadPropertyStore> {
Arc::new(DeadPropertyStore::new(pool))
}
@@ -32,6 +32,14 @@ const MAX_LOCK_TIMEOUT_SECS: u64 = 86_400; // 24 hours
pub struct LockEntry {
pub info: LockInfo,
pub path: String,
/// The user who acquired the lock. `None` for entries seeded by
/// unit tests or refresh paths that don't carry a caller (the
/// refresh flow rebuilds from the existing entry without a new
/// caller context, so we preserve whatever was there). RFC 4918
/// §9.11's "MUST be requested by the owner" rule for UNLOCK is
/// enforced by comparing this against the caller in
/// `handle_unlock`.
pub caller_user_id: Option<uuid::Uuid>,
}
/// Per-entry expiration policy for the `by_path` cache.
@@ -106,20 +114,39 @@ impl WebDavLockStore {
/// Attempt to acquire a lock on `path`.
///
/// Returns `Ok(LockEntry)` on success, or `Err(existing)` if the resource
/// is already exclusively locked by a different token.
/// Returns `Ok(LockEntry)` on success, or `Err(existing)` when:
/// - The existing lock is exclusive (blocks any new lock), or
/// - The new lock is exclusive and any lock already exists (RFC 4918 §7.8).
#[allow(clippy::result_large_err)]
pub fn acquire(&self, path: &str, info: LockInfo) -> Result<LockEntry, LockEntry> {
// Check for existing conflicting lock
if let Some(existing) = self.by_path.get(path)
&& existing.info.scope == LockScope::Exclusive
{
return Err(existing);
pub fn acquire(
&self,
path: &str,
info: LockInfo,
caller_user_id: Option<uuid::Uuid>,
) -> Result<LockEntry, LockEntry> {
if let Some(existing) = self.by_path.get(path) {
// Exclusive existing lock → blocks everything.
// New exclusive lock → blocked by any existing lock (shared or exclusive).
if existing.info.scope == LockScope::Exclusive || info.scope == LockScope::Exclusive {
return Err(existing);
}
// Both shared: keep the first holder as the enforcement sentinel in
// `by_path` so releasing a secondary holder cannot clear the lock.
// Register the new token only in the reverse index so UNLOCK works.
let entry = LockEntry {
info,
path: path.to_owned(),
caller_user_id,
};
self.by_token
.insert(entry.info.token.clone(), path.to_owned());
return Ok(entry);
}
let entry = LockEntry {
info,
path: path.to_owned(),
caller_user_id,
};
// `LockExpiry` derives the TTL from `entry.info.timeout` on insert —
@@ -242,6 +269,7 @@ mod tests {
LockEntry {
info: lock_info(token, timeout, LockScope::Exclusive),
path: "/file.txt".to_owned(),
caller_user_id: None,
}
}
@@ -293,7 +321,7 @@ mod tests {
let store = WebDavLockStore::new(16);
let info = lock_info("urn:token-1", Some("Second-600"), LockScope::Exclusive);
let acquired = store.acquire("/a.txt", info).expect("acquire");
let acquired = store.acquire("/a.txt", info, None).expect("acquire");
assert_eq!(acquired.info.token, "urn:token-1");
// Resolvable by both indexes.
@@ -320,12 +348,14 @@ mod tests {
.acquire(
"/a.txt",
lock_info("urn:token-1", Some("Second-600"), LockScope::Exclusive),
None,
)
.expect("first acquire");
let conflict = store.acquire(
"/a.txt",
lock_info("urn:token-2", Some("Second-600"), LockScope::Exclusive),
None,
);
assert!(conflict.is_err());
// The original holder is returned so the caller can report it.
@@ -339,6 +369,7 @@ mod tests {
.acquire(
"/a.txt",
lock_info("urn:token-1", Some("Infinite"), LockScope::Exclusive),
None,
)
.expect("acquire");
@@ -203,7 +203,10 @@ impl WopiDiscoveryService {
for attr in e.attributes().flatten() {
let value = attr
.decode_and_unescape_value(reader.decoder())
.decoded_and_normalized_value(
quick_xml::XmlVersion::Implicit1_0,
reader.decoder(),
)
.map(|value| value.into_owned())
.unwrap_or_else(|_| String::from_utf8_lossy(&attr.value).to_string());
+131 -62
View File
@@ -44,15 +44,22 @@ impl From<ZipError> for DomainError {
}
}
/// Type alias for the fully-async ZIP writer backed by a buffered tokio file.
type AsyncZipWriter = ZipFileWriter<Compat<BufWriter<tokio::fs::File>>>;
/// Fully-async ZIP writer over any buffered tokio sink (temp file for the
/// legacy path, one half of a `tokio::io::duplex` for the streaming path).
type AsyncZipWriter<W> = ZipFileWriter<Compat<BufWriter<W>>>;
/// One planned archive entry, in final ZIP order.
enum ZipPlanEntry {
/// Directory entry (Stored, zero-length body).
Dir(String),
/// File entry: ZIP-relative path + file id to stream from the blob store.
File { zip_path: String, file_id: String },
/// `compression` is picked from the file's MIME type at plan time —
/// `Stored` for already-compressed media (JPEG/MP4/…), `Deflate` otherwise.
File {
zip_path: String,
file_id: String,
compression: Compression,
},
}
/// Message protocol from the prefetch task to the ZIP writer. For each
@@ -74,8 +81,11 @@ const PREFETCH_BUFFER_CHUNKS: usize = 64;
///
/// Uses `async_zip` for fully-async archive creation. Every write (headers,
/// compressed chunk data, central directory) goes through
/// `tokio::io::BufWriter` → `tokio::fs::File`, so **no Tokio worker is ever
/// blocked** by disk I/O or compression.
/// `tokio::io::BufWriter` → `tokio::fs::File`, so no Tokio worker is ever
/// blocked by disk I/O. Deflate itself DOES run inline on the writing task
/// (async_zip compresses inside `poll_write`), which is why entries whose
/// MIME says the content is already compressed are `Stored` instead — that
/// turns the archive hot path from ~1 CPU core per download into CRC + memcpy.
///
/// Archive creation is a 2-stage pipeline: a prefetch task reads file
/// content from the blob store ahead of the writer, so the next file's
@@ -110,6 +120,110 @@ impl ZipService {
folder_id: &str,
folder_name: &str,
) -> Result<NamedTempFile> {
let plan = self.plan_archive(folder_id, folder_name).await?;
// ── Open the temp file + ZIP writer ──────────────────────────────
let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
let tokio_file = tokio::fs::File::create(temp.path())
.await
.map_err(ZipError::IoError)?;
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
let _prefetcher = tokio::spawn(Self::prefetch_files(
self.file_service.clone(),
Self::planned_file_ids(&plan),
tx,
));
Self::write_archive(tokio_file, &plan, &mut rx).await?;
Ok(temp)
}
/// Streaming variant: the archive bytes are produced on a spawned task
/// and yielded as they are written — the client's first byte arrives
/// after the first entry starts, not after the whole archive has been
/// built (the temp-file variant's time-to-first-byte grows with folder
/// size; benches/ZIP-STREAM.md). The plan phase still runs inline so
/// planning errors surface as proper HTTP errors; a blob-read error
/// mid-archive can only truncate the stream (no central directory →
/// clients detect the corrupt archive), which is the standard tradeoff
/// for streamed ZIPs.
pub async fn create_folder_zip_stream(
&self,
folder_id: &str,
folder_name: &str,
) -> Result<impl futures::Stream<Item = std::io::Result<bytes::Bytes>> + Send + use<>> {
let plan = self.plan_archive(folder_id, folder_name).await?;
let (writer, reader) = tokio::io::duplex(256 * 1024);
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
let _prefetcher = tokio::spawn(Self::prefetch_files(
self.file_service.clone(),
Self::planned_file_ids(&plan),
tx,
));
tokio::spawn(async move {
if let Err(e) = Self::write_archive(writer, &plan, &mut rx).await {
// Dropping the writer EOFs the reader early — the truncated
// archive has no central directory, so clients flag it.
warn!("Streaming ZIP aborted mid-archive: {e}");
}
});
Ok(tokio_util::io::ReaderStream::new(reader))
}
/// File ids of the plan, in archive order (the prefetcher's read list).
fn planned_file_ids(plan: &[ZipPlanEntry]) -> Vec<String> {
plan.iter()
.filter_map(|entry| match entry {
ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()),
ZipPlanEntry::Dir(_) => None,
})
.collect()
}
/// Write every planned entry through a buffered ZIP writer over `sink`,
/// then finalize (central directory + flush). Shared by the temp-file
/// and streaming variants.
async fn write_archive<W: tokio::io::AsyncWrite + Unpin>(
sink: W,
plan: &[ZipPlanEntry],
rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,
) -> Result<()> {
let buf_writer = BufWriter::with_capacity(256 * 1024, sink);
let mut zip = ZipFileWriter::with_tokio(buf_writer);
for entry in plan {
match entry {
ZipPlanEntry::Dir(zip_dir) => {
let dir_entry =
ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
match zip.write_entry_whole(dir_entry, &[]).await {
Ok(()) => debug!("Folder added to ZIP: {}", zip_dir),
Err(e) => {
warn!("Could not add folder entry (may already exist): {}", e);
}
}
}
ZipPlanEntry::File {
zip_path,
compression,
..
} => {
Self::write_prefetched_file(&mut zip, zip_path, *compression, rx).await?;
}
}
}
let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
compat_writer.close().await.map_err(ZipError::IoError)?;
Ok(())
}
/// Resolve the folder, fetch its subtree (2 bulk queries) and lay out
/// the archive entries in final ZIP order.
async fn plan_archive(&self, folder_id: &str, folder_name: &str) -> Result<Vec<ZipPlanEntry>> {
info!(
"Creating ZIP for folder: {} (ID: {})",
folder_name, folder_id
@@ -183,62 +297,15 @@ impl ZipService {
plan.push(ZipPlanEntry::File {
zip_path: format!("{}{}", zip_dir, file.name),
file_id: file.id.to_string(),
compression: crate::common::mime_detect::zip_entry_compression(
&file.mime_type,
),
});
}
}
}
// ── 5. Open the temp file + ZIP writer ───────────────────────────
let temp = NamedTempFile::new().map_err(ZipError::IoError)?;
let tokio_file = tokio::fs::File::create(temp.path())
.await
.map_err(ZipError::IoError)?;
let buf_writer = BufWriter::with_capacity(256 * 1024, tokio_file);
let mut zip = ZipFileWriter::with_tokio(buf_writer);
// ── 6. Write entries: 2-stage pipeline ───────────────────────────
// The prefetch task reads blob streams for the planned files, in
// order, ahead of the writer — the next file's blob-store latency
// overlaps the current file's deflate. If the writer bails out,
// dropping the receiver makes the prefetcher's next send fail and
// it stops on its own.
let file_ids: Vec<String> = plan
.iter()
.filter_map(|entry| match entry {
ZipPlanEntry::File { file_id, .. } => Some(file_id.clone()),
ZipPlanEntry::Dir(_) => None,
})
.collect();
let (tx, mut rx) = tokio::sync::mpsc::channel::<Prefetched>(PREFETCH_BUFFER_CHUNKS);
let _prefetcher = tokio::spawn(Self::prefetch_files(
self.file_service.clone(),
file_ids,
tx,
));
for entry in &plan {
match entry {
ZipPlanEntry::Dir(zip_dir) => {
let dir_entry =
ZipEntryBuilder::new(zip_dir.clone().into(), Compression::Stored);
match zip.write_entry_whole(dir_entry, &[]).await {
Ok(()) => debug!("Folder added to ZIP: {}", zip_dir),
Err(e) => {
warn!("Could not add folder entry (may already exist): {}", e);
}
}
}
ZipPlanEntry::File { zip_path, .. } => {
Self::write_prefetched_file(&mut zip, zip_path, &mut rx).await?;
}
}
}
// ── 7. Finalize ──────────────────────────────────────────────────
let mut compat_writer = zip.close().await.map_err(ZipError::AsyncZipError)?;
compat_writer.close().await.map_err(ZipError::IoError)?;
Ok(temp)
Ok(plan)
}
/// Prefetch stage: streams each planned file's content from the blob
@@ -282,17 +349,19 @@ impl ZipService {
}
}
/// Writer stage: drains one file's prefetched chunks into a Deflate
/// ZIP entry. Peak memory stays bounded by the channel, independent
/// of individual file sizes.
async fn write_prefetched_file(
zip: &mut AsyncZipWriter,
/// Writer stage: drains one file's prefetched chunks into a ZIP entry
/// (`Stored` for already-compressed media, `Deflate` otherwise — see
/// `entry_compression`). Peak memory stays bounded by the channel,
/// independent of individual file sizes.
async fn write_prefetched_file<W: tokio::io::AsyncWrite + Unpin>(
zip: &mut AsyncZipWriter<W>,
zip_path: &str,
compression: Compression,
rx: &mut tokio::sync::mpsc::Receiver<Prefetched>,
) -> Result<()> {
info!("Adding file to ZIP: {}", zip_path);
let entry = ZipEntryBuilder::new(zip_path.to_string().into(), Compression::Deflate);
let entry = ZipEntryBuilder::new(zip_path.to_string().into(), compression);
let mut entry_writer = zip
.write_entry_stream(entry)
.await