Merge pull request #453: Legacy blob re-chunk migration + DAV multiget optimizations
Legacy whole-file blob re-chunk migration + DAV multiget optimizations
This commit is contained in:
+2
-2
@@ -1,5 +1,5 @@
|
||||
# ─── Stage 1: Shared build base (avoids duplicate apk install) ────────────────
|
||||
FROM rust:1.94.1-alpine3.23 AS base
|
||||
FROM rust:1.96-alpine3.24 AS base
|
||||
# sqlx's postgres driver speaks the wire protocol in pure Rust (no pq-sys in
|
||||
# Cargo.lock) and TLS goes through rustls, so libpq headers are never needed at
|
||||
# build time. perl/make/gcc/musl-dev remain for the C builds of aws-lc-sys.
|
||||
@@ -41,7 +41,7 @@ ARG DATABASE_URL="postgres://postgres:postgres@localhost/oxicloud"
|
||||
RUN DATABASE_URL="${DATABASE_URL}" cargo build --release
|
||||
|
||||
# ─── Stage 4: Minimal runtime image ──────────────────────────────────────────
|
||||
FROM alpine:3.23.3
|
||||
FROM alpine:3.24.0
|
||||
|
||||
# OCI image metadata
|
||||
LABEL org.opencontainers.image.title="OxiCloud" \
|
||||
|
||||
@@ -87,6 +87,15 @@ OXICLOUD_SERVER_HOST=127.0.0.1
|
||||
# fresher sync detection, higher = fewer background UPDATEs. Minimum: 100.
|
||||
#OXICLOUD_TREE_ETAG_FLUSH_MS=500
|
||||
|
||||
# One-time startup migration that converts pre-CDC whole-file blobs (files
|
||||
# uploaded before chunked dedup landed) into CDC chunk manifests, in the
|
||||
# background on the maintenance pool. Fixes the legacy penalty where a Range
|
||||
# read (video seek) reads — and with encryption, DECRYPTS — the entire blob.
|
||||
# Idempotent; a no-op once no legacy blobs remain. Disable only on metered
|
||||
# remote backends (S3/Azure egress) where the one-time re-read of every
|
||||
# legacy blob should be scheduled deliberately, e.g. off-peak.
|
||||
#OXICLOUD_LEGACY_RECHUNK=true
|
||||
|
||||
# Allow multiple processes to bind to the same port (SO_REUSEPORT).
|
||||
# DISABLED by default — leaving this off means a second accidental instance
|
||||
# will fail immediately with "address already in use", which is the safe behaviour.
|
||||
|
||||
@@ -8,3 +8,91 @@ pub mod webdav_adapter;
|
||||
mod caldav_adapter_test;
|
||||
#[cfg(test)]
|
||||
mod carddav_adapter_test;
|
||||
|
||||
/// Extract the resource UID from a DAV multiget `href`.
|
||||
///
|
||||
/// CalDAV/CardDAV multiget REPORTs address object resources by full href
|
||||
/// (e.g. `/caldav/{calendar_id}/{uid}.ics`, possibly with a username
|
||||
/// segment). The UID is the last path segment with the protocol
|
||||
/// `extension` (`.ics` / `.vcf`, matched case-insensitively) stripped,
|
||||
/// then percent-decoded — hrefs arrive in the XML body, so they have not
|
||||
/// gone through URL-path decoding, and clients may re-encode hrefs they
|
||||
/// previously read from the server.
|
||||
///
|
||||
/// Returns `None` for collection hrefs (empty last segment) or segments
|
||||
/// that are not valid UTF-8 after decoding.
|
||||
pub fn uid_from_multiget_href(href: &str, extension: &str) -> Option<String> {
|
||||
// A trailing slash denotes a collection, not an object resource.
|
||||
if href.ends_with('/') {
|
||||
return None;
|
||||
}
|
||||
let segment = href.rsplit('/').next()?;
|
||||
|
||||
// Case-insensitive ASCII extension strip; the matched tail is ASCII,
|
||||
// so the byte cut is guaranteed to land on a char boundary.
|
||||
let bytes = segment.as_bytes();
|
||||
let ext = extension.as_bytes();
|
||||
let segment =
|
||||
if bytes.len() >= ext.len() && bytes[bytes.len() - ext.len()..].eq_ignore_ascii_case(ext) {
|
||||
&segment[..segment.len() - ext.len()]
|
||||
} else {
|
||||
segment
|
||||
};
|
||||
|
||||
let decoded = percent_encoding::percent_decode_str(segment)
|
||||
.decode_utf8()
|
||||
.ok()?;
|
||||
let uid = decoded.trim();
|
||||
(!uid.is_empty()).then(|| uid.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod multiget_href_tests {
|
||||
use super::uid_from_multiget_href;
|
||||
|
||||
#[test]
|
||||
fn plain_caldav_href() {
|
||||
assert_eq!(
|
||||
uid_from_multiget_href("/caldav/abc-123/event-uid.ics", ".ics"),
|
||||
Some("event-uid".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn href_with_username_prefix() {
|
||||
assert_eq!(
|
||||
uid_from_multiget_href("/carddav/alice/book-1/uid-42.vcf", ".vcf"),
|
||||
Some("uid-42".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uppercase_extension() {
|
||||
assert_eq!(
|
||||
uid_from_multiget_href("/caldav/abc/EVENT.ICS", ".ics"),
|
||||
Some("EVENT".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percent_encoded_uid() {
|
||||
assert_eq!(
|
||||
uid_from_multiget_href("/caldav/abc/uid%40example.com.ics", ".ics"),
|
||||
Some("uid@example.com".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_extension_uses_whole_segment() {
|
||||
assert_eq!(
|
||||
uid_from_multiget_href("/caldav/abc/bare-uid", ".ics"),
|
||||
Some("bare-uid".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collection_href_yields_none() {
|
||||
assert_eq!(uid_from_multiget_href("/caldav/abc/", ".ics"), None);
|
||||
assert_eq!(uid_from_multiget_href("", ".ics"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ use crate::domain::entities::app_password::AppPassword;
|
||||
use crate::domain::entities::device_code::DeviceCode;
|
||||
use crate::domain::entities::session::Session;
|
||||
use crate::domain::entities::user::User;
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
// ============================================================================
|
||||
@@ -51,8 +52,14 @@ pub trait TokenServicePort: Send + Sync + 'static {
|
||||
/// Generate an access token for a user
|
||||
fn generate_access_token(&self, user: &User) -> Result<String, DomainError>;
|
||||
|
||||
/// Validate a token and extract its claims
|
||||
fn validate_token(&self, token: &str) -> Result<TokenClaims, DomainError>;
|
||||
/// Validate a token and extract its claims.
|
||||
///
|
||||
/// Returns `Arc<TokenClaims>` so the implementation's validation cache can
|
||||
/// hand back a hot entry with a refcount bump instead of deep-cloning the
|
||||
/// (multi-`String`) claims on every authenticated request. Callers that
|
||||
/// only read fields go through `Deref`; the few that retain a field clone
|
||||
/// just that one.
|
||||
fn validate_token(&self, token: &str) -> Result<Arc<TokenClaims>, DomainError>;
|
||||
|
||||
/// Generate a refresh token
|
||||
fn generate_refresh_token(&self) -> String;
|
||||
|
||||
@@ -95,6 +95,14 @@ pub trait CalendarStoragePort: Send + Sync + 'static {
|
||||
calendar_id: &str,
|
||||
ical_uid: &str,
|
||||
) -> Result<Option<CalendarEventDto>, DomainError>;
|
||||
/// Indexed batch lookup by iCalendar UID (`ical_uid = ANY(...)`) — the
|
||||
/// CalDAV multiget REPORT must use this instead of listing the whole
|
||||
/// calendar (every row + its `ical_data`) and filtering client-side.
|
||||
async fn find_events_by_ical_uids(
|
||||
&self,
|
||||
calendar_id: &str,
|
||||
ical_uids: &[String],
|
||||
) -> Result<Vec<CalendarEventDto>, DomainError>;
|
||||
async fn list_events_by_calendar(
|
||||
&self,
|
||||
calendar_id: &str,
|
||||
@@ -197,6 +205,15 @@ pub trait CalendarUseCase: Send + Sync + 'static {
|
||||
ical_uid: &str,
|
||||
user_id: Uuid,
|
||||
) -> Result<Option<CalendarEventDto>, DomainError>;
|
||||
/// Resolve a batch of events by their iCalendar UIDs with a single
|
||||
/// indexed query. UIDs without a matching event are silently absent
|
||||
/// from the result (CalDAV multiget semantics).
|
||||
async fn get_events_by_ical_uids(
|
||||
&self,
|
||||
calendar_id: &str,
|
||||
ical_uids: &[String],
|
||||
user_id: Uuid,
|
||||
) -> Result<Vec<CalendarEventDto>, DomainError>;
|
||||
async fn list_events(
|
||||
&self,
|
||||
calendar_id: &str,
|
||||
|
||||
@@ -82,9 +82,25 @@ pub trait ContactUseCase: Send + Sync + 'static {
|
||||
uid: &str,
|
||||
user_id: Uuid,
|
||||
) -> Result<Option<ContactDto>, DomainError>;
|
||||
/// Resolve a batch of contacts by their vCard UIDs with a single
|
||||
/// indexed query (`uid = ANY(...)`) — the CardDAV multiget REPORT
|
||||
/// must use this instead of listing the whole address book and
|
||||
/// filtering client-side. UIDs without a matching contact are
|
||||
/// silently absent from the result.
|
||||
async fn get_contacts_by_uids(
|
||||
&self,
|
||||
address_book_id: &str,
|
||||
uids: &[String],
|
||||
user_id: Uuid,
|
||||
) -> Result<Vec<ContactDto>, DomainError>;
|
||||
/// List contacts in an address book. `limit`/`offset` bound the
|
||||
/// result for paginated callers (REST API); `None` returns the full
|
||||
/// book, which the CardDAV listing/sync paths rely on.
|
||||
async fn list_contacts(
|
||||
&self,
|
||||
address_book_id: &str,
|
||||
limit: Option<i64>,
|
||||
offset: Option<i64>,
|
||||
user_id: Uuid,
|
||||
) -> Result<Vec<ContactDto>, DomainError>;
|
||||
async fn search_contacts(
|
||||
|
||||
@@ -30,9 +30,25 @@ const NC_APP_PASSWORD_GROUP_LEN: usize = 5;
|
||||
const NC_PREFIX_LEN: usize = 8;
|
||||
|
||||
/// TTL for cached Basic Auth verification results.
|
||||
/// Balances performance (avoids repeated Argon2id + DB queries) with security
|
||||
/// (limits the window during which a revoked app password remains usable).
|
||||
const BASIC_AUTH_CACHE_TTL_SECS: u64 = 30;
|
||||
///
|
||||
/// DAV sync clients (Nautilus, Windows Explorer, Apple Calendar, …) poll
|
||||
/// continuously, and every cache miss costs a full Argon2id verification
|
||||
/// (~50–100 ms of CPU) plus two DB round-trips. A 30 s TTL re-paid that
|
||||
/// cost every 30 s per client; 5 min cuts it ~10× under steady sync load.
|
||||
///
|
||||
/// Security envelope of this window:
|
||||
/// - **Revocation is immediate**: `revoke()` calls `invalidate_entries_if`
|
||||
/// on this cache for the user, so a revoked password never survives in
|
||||
/// cache regardless of TTL.
|
||||
/// - **Expiry / deactivation are bounded by the TTL**: `expires_at` and
|
||||
/// `user.is_active()` are only re-checked on a cache *miss* (the DB
|
||||
/// query filters them), so an app password that expires — or a user
|
||||
/// deactivated via `set_user_active` — may keep authenticating from
|
||||
/// cache for at most this long. 5 min is comparable to a typical JWT
|
||||
/// access-token lifetime, so the grace window is consistent across
|
||||
/// auth surfaces. Lengthen with care; shorten if a tighter bound on
|
||||
/// post-deactivation access is required.
|
||||
const BASIC_AUTH_CACHE_TTL_SECS: u64 = 300;
|
||||
|
||||
/// Maximum number of cached Basic Auth verifications.
|
||||
/// Each entry is ~160 bytes (32-byte key + 4 small strings), so 10 000
|
||||
@@ -61,9 +77,9 @@ pub struct AppPasswordService {
|
||||
///
|
||||
/// **Value**: the authenticated identity (user_id, username, email, role).
|
||||
///
|
||||
/// **Eviction**: TTL-based (30 s) + capacity-based (10 000 entries).
|
||||
/// Failed verifications are *never* cached, so brute-force attackers
|
||||
/// always pay the full Argon2id cost.
|
||||
/// **Eviction**: TTL-based (see `BASIC_AUTH_CACHE_TTL_SECS`) +
|
||||
/// capacity-based (10 000 entries). Failed verifications are *never*
|
||||
/// cached, so brute-force attackers always pay the full Argon2id cost.
|
||||
auth_cache: Cache<[u8; 32], CachedBasicAuthResult>,
|
||||
}
|
||||
|
||||
|
||||
@@ -300,6 +300,32 @@ impl CalendarUseCase for CalendarService {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_events_by_ical_uids(
|
||||
&self,
|
||||
calendar_id: &str,
|
||||
ical_uids: &[String],
|
||||
user_id: Uuid,
|
||||
) -> Result<Vec<CalendarEventDto>, DomainError> {
|
||||
let has_access = self
|
||||
.calendar_storage
|
||||
.check_calendar_access(calendar_id, user_id)
|
||||
.await?;
|
||||
let calendar = self.calendar_storage.get_calendar(calendar_id).await?;
|
||||
if !has_access && !calendar.is_public {
|
||||
return Err(DomainError::new(
|
||||
ErrorKind::AccessDenied,
|
||||
"Calendar",
|
||||
"You don't have permission to view events in this calendar",
|
||||
));
|
||||
}
|
||||
if ical_uids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.calendar_storage
|
||||
.find_events_by_ical_uids(calendar_id, ical_uids)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_events(
|
||||
&self,
|
||||
calendar_id: &str,
|
||||
|
||||
@@ -796,9 +796,34 @@ impl ContactUseCase for ContactService {
|
||||
Ok(contact.map(ContactDto::from))
|
||||
}
|
||||
|
||||
async fn get_contacts_by_uids(
|
||||
&self,
|
||||
address_book_id: &str,
|
||||
uids: &[String],
|
||||
user_id: Uuid,
|
||||
) -> Result<Vec<ContactDto>, DomainError> {
|
||||
let id = Uuid::parse_str(address_book_id)
|
||||
.map_err(|_| DomainError::validation_error("Invalid address book ID format"))?;
|
||||
|
||||
// Check if user has access to the address book
|
||||
self.check_address_book_access(&id, &user_id).await?;
|
||||
|
||||
if uids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let contacts = self
|
||||
.contact_repository
|
||||
.get_contacts_by_uids(&id, uids)
|
||||
.await?;
|
||||
Ok(contacts.into_iter().map(ContactDto::from).collect())
|
||||
}
|
||||
|
||||
async fn list_contacts(
|
||||
&self,
|
||||
address_book_id: &str,
|
||||
limit: Option<i64>,
|
||||
offset: Option<i64>,
|
||||
user_id: Uuid,
|
||||
) -> Result<Vec<ContactDto>, DomainError> {
|
||||
let id = Uuid::parse_str(address_book_id)
|
||||
@@ -808,10 +833,17 @@ impl ContactUseCase for ContactService {
|
||||
self.check_address_book_access(&id, &user_id).await?;
|
||||
|
||||
// Get contacts
|
||||
let contacts = self
|
||||
.contact_repository
|
||||
.get_contacts_by_address_book(&id)
|
||||
.await?;
|
||||
let contacts = if limit.is_some() || offset.is_some() {
|
||||
let limit = limit.unwrap_or(100);
|
||||
let offset = offset.unwrap_or(0);
|
||||
self.contact_repository
|
||||
.get_contacts_by_address_book_paginated(&id, limit, offset)
|
||||
.await?
|
||||
} else {
|
||||
self.contact_repository
|
||||
.get_contacts_by_address_book(&id)
|
||||
.await?
|
||||
};
|
||||
let dtos = contacts.into_iter().map(ContactDto::from).collect();
|
||||
|
||||
Ok(dtos)
|
||||
@@ -1324,7 +1356,9 @@ impl StorageUseCase for ContactService {
|
||||
let user_id = Uuid::parse_str(user_id)
|
||||
.map_err(|_| DomainError::validation_error("Invalid user_id format"))?;
|
||||
|
||||
let result = self.list_contacts(address_book_id, user_id).await?;
|
||||
let result = self
|
||||
.list_contacts(address_book_id, None, None, user_id)
|
||||
.await?;
|
||||
Ok(serde_json::to_value(result).unwrap())
|
||||
}
|
||||
"search_contacts" => {
|
||||
|
||||
@@ -254,6 +254,14 @@ pub struct StorageConfig {
|
||||
/// bound on how stale an ancestor folder's ETag can be after a change.
|
||||
/// Default: 500. Env: `OXICLOUD_TREE_ETAG_FLUSH_MS`.
|
||||
pub tree_etag_flush_ms: u64,
|
||||
/// Startup background migration that re-chunks legacy whole-file blobs
|
||||
/// (written before CDC chunking landed) into chunk manifests, so Range
|
||||
/// reads stop paying a full-blob read — and, with encryption enabled, a
|
||||
/// full-blob decrypt. Idempotent and incremental; a no-op (one COUNT
|
||||
/// query) once no legacy blobs remain. Disable on metered remote
|
||||
/// backends where the one-time re-read of every legacy blob should be
|
||||
/// scheduled deliberately. Default: true. Env: `OXICLOUD_LEGACY_RECHUNK`.
|
||||
pub legacy_rechunk_enabled: bool,
|
||||
/// Which blob storage backend to use (`local`, `s3`, or `azure`).
|
||||
pub backend: StorageBackendType,
|
||||
/// S3-compatible backend configuration (used when `backend == S3`).
|
||||
@@ -397,6 +405,7 @@ impl Default for StorageConfig {
|
||||
chunk_dir: None,
|
||||
usage_reconcile_secs: 600, // 10 minutes
|
||||
tree_etag_flush_ms: 500,
|
||||
legacy_rechunk_enabled: true,
|
||||
backend: StorageBackendType::Local,
|
||||
s3: None,
|
||||
azure: None,
|
||||
@@ -1305,6 +1314,12 @@ impl AppConfig {
|
||||
config.storage.tree_etag_flush_ms = val;
|
||||
}
|
||||
|
||||
// Legacy whole-file blob re-chunk migration (startup background task)
|
||||
if let Ok(enabled) = env::var("OXICLOUD_LEGACY_RECHUNK") {
|
||||
config.storage.legacy_rechunk_enabled =
|
||||
enabled.eq_ignore_ascii_case("true") || enabled == "1";
|
||||
}
|
||||
|
||||
// Storage backend selection
|
||||
if let Ok(backend) = env::var("OXICLOUD_STORAGE_BACKEND") {
|
||||
match backend.to_lowercase().as_str() {
|
||||
|
||||
@@ -319,6 +319,18 @@ impl AppServiceFactory {
|
||||
);
|
||||
dedup_service.initialize().await?;
|
||||
|
||||
// One-time background migration: re-chunk pre-CDC whole-file blobs
|
||||
// into chunk manifests so Range reads (and, with encryption, partial
|
||||
// decrypts) stop paying for the entire blob. No-op once converged.
|
||||
if self.config.storage.legacy_rechunk_enabled {
|
||||
dedup_service.spawn_legacy_rechunk();
|
||||
} else {
|
||||
tracing::info!(
|
||||
"Legacy re-chunk migration disabled (OXICLOUD_LEGACY_RECHUNK=false) — \
|
||||
pre-CDC whole-file blobs, if any, will keep using the legacy read path"
|
||||
);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Core services initialized: path service, file content cache, thumbnails, chunked upload, image transcode, dedup (PRIMARY blob storage)"
|
||||
);
|
||||
|
||||
@@ -53,6 +53,16 @@ pub trait CalendarEventRepository: Send + Sync + 'static {
|
||||
ical_uid: &str,
|
||||
) -> CalendarEventRepositoryResult<Option<CalendarEvent>>;
|
||||
|
||||
/// Finds the events matching any of the given iCalendar UIDs in one
|
||||
/// indexed query (`ical_uid = ANY(...)`). Used by CalDAV multiget so a
|
||||
/// request for a handful of events never pays for the whole calendar.
|
||||
/// UIDs with no matching event are silently absent from the result.
|
||||
async fn find_events_by_ical_uids(
|
||||
&self,
|
||||
calendar_id: &Uuid,
|
||||
ical_uids: &[String],
|
||||
) -> CalendarEventRepositoryResult<Vec<CalendarEvent>>;
|
||||
|
||||
/// Counts events in a calendar
|
||||
async fn count_events_in_calendar(
|
||||
&self,
|
||||
|
||||
@@ -16,10 +16,27 @@ pub trait ContactRepository: Send + Sync + 'static {
|
||||
address_book_id: &Uuid,
|
||||
uid: &str,
|
||||
) -> ContactRepositoryResult<Option<Contact>>;
|
||||
/// Fetches the contacts matching any of the given vCard UIDs in one
|
||||
/// indexed query (`uid = ANY(...)`). Used by CardDAV multiget so a
|
||||
/// request for a handful of contacts never pays for the whole book.
|
||||
/// UIDs with no matching contact are silently absent from the result.
|
||||
async fn get_contacts_by_uids(
|
||||
&self,
|
||||
address_book_id: &Uuid,
|
||||
uids: &[String],
|
||||
) -> ContactRepositoryResult<Vec<Contact>>;
|
||||
async fn get_contacts_by_address_book(
|
||||
&self,
|
||||
address_book_id: &Uuid,
|
||||
) -> ContactRepositoryResult<Vec<Contact>>;
|
||||
/// Same as [`Self::get_contacts_by_address_book`] but bounded by
|
||||
/// `LIMIT`/`OFFSET` for paginated listings.
|
||||
async fn get_contacts_by_address_book_paginated(
|
||||
&self,
|
||||
address_book_id: &Uuid,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> ContactRepositoryResult<Vec<Contact>>;
|
||||
async fn get_contacts_by_email(&self, email: &str) -> ContactRepositoryResult<Vec<Contact>>;
|
||||
async fn get_contacts_by_group(&self, group_id: &Uuid)
|
||||
-> ContactRepositoryResult<Vec<Contact>>;
|
||||
|
||||
@@ -422,6 +422,26 @@ impl CalendarStoragePort for CalendarStorageAdapter {
|
||||
Ok(event.map(CalendarEventDto::from))
|
||||
}
|
||||
|
||||
async fn find_events_by_ical_uids(
|
||||
&self,
|
||||
calendar_id: &str,
|
||||
ical_uids: &[String],
|
||||
) -> Result<Vec<CalendarEventDto>, DomainError> {
|
||||
let uuid = Uuid::parse_str(calendar_id).map_err(|_| {
|
||||
DomainError::new(
|
||||
ErrorKind::InvalidInput,
|
||||
"Calendar",
|
||||
"Invalid calendar ID format",
|
||||
)
|
||||
})?;
|
||||
|
||||
let events = self
|
||||
.event_repository
|
||||
.find_events_by_ical_uids(&uuid, ical_uids)
|
||||
.await?;
|
||||
Ok(events.into_iter().map(CalendarEventDto::from).collect())
|
||||
}
|
||||
|
||||
async fn list_events_by_calendar(
|
||||
&self,
|
||||
calendar_id: &str,
|
||||
|
||||
@@ -730,9 +730,10 @@ impl ContactUseCase for ContactStorageAdapter {
|
||||
Ok(contact.map(ContactDto::from))
|
||||
}
|
||||
|
||||
async fn list_contacts(
|
||||
async fn get_contacts_by_uids(
|
||||
&self,
|
||||
address_book_id: &str,
|
||||
uids: &[String],
|
||||
user_id: Uuid,
|
||||
) -> Result<Vec<ContactDto>, DomainError> {
|
||||
let uuid = Self::parse_uuid(address_book_id, "AddressBook")?;
|
||||
@@ -740,13 +741,43 @@ impl ContactUseCase for ContactStorageAdapter {
|
||||
// Check read access
|
||||
self.check_address_book_access(&uuid, user_id).await?;
|
||||
|
||||
if uids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let contacts = self
|
||||
.contact_repository
|
||||
.get_contacts_by_address_book(&uuid)
|
||||
.get_contacts_by_uids(&uuid, uids)
|
||||
.await?;
|
||||
Ok(contacts.into_iter().map(ContactDto::from).collect())
|
||||
}
|
||||
|
||||
async fn list_contacts(
|
||||
&self,
|
||||
address_book_id: &str,
|
||||
limit: Option<i64>,
|
||||
offset: Option<i64>,
|
||||
user_id: Uuid,
|
||||
) -> Result<Vec<ContactDto>, DomainError> {
|
||||
let uuid = Self::parse_uuid(address_book_id, "AddressBook")?;
|
||||
|
||||
// Check read access
|
||||
self.check_address_book_access(&uuid, user_id).await?;
|
||||
|
||||
let contacts = if limit.is_some() || offset.is_some() {
|
||||
let limit = limit.unwrap_or(100);
|
||||
let offset = offset.unwrap_or(0);
|
||||
self.contact_repository
|
||||
.get_contacts_by_address_book_paginated(&uuid, limit, offset)
|
||||
.await?
|
||||
} else {
|
||||
self.contact_repository
|
||||
.get_contacts_by_address_book(&uuid)
|
||||
.await?
|
||||
};
|
||||
Ok(contacts.into_iter().map(ContactDto::from).collect())
|
||||
}
|
||||
|
||||
async fn search_contacts(
|
||||
&self,
|
||||
address_book_id: &str,
|
||||
|
||||
@@ -370,6 +370,56 @@ impl CalendarEventRepository for CalendarEventPgRepository {
|
||||
}
|
||||
}
|
||||
|
||||
async fn find_events_by_ical_uids(
|
||||
&self,
|
||||
calendar_id: &Uuid,
|
||||
ical_uids: &[String],
|
||||
) -> CalendarEventRepositoryResult<Vec<CalendarEvent>> {
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT
|
||||
id, calendar_id, summary, description, location,
|
||||
start_time, end_time, all_day, rrule,
|
||||
created_at, updated_at, ical_uid, ical_data
|
||||
FROM caldav.calendar_events
|
||||
WHERE calendar_id = $1 AND ical_uid = ANY($2)
|
||||
ORDER BY start_time
|
||||
"#,
|
||||
)
|
||||
.bind(calendar_id)
|
||||
.bind(ical_uids)
|
||||
.fetch_all(&*self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::database_error(format!("Failed to get calendar events by UIDs: {}", e))
|
||||
})?;
|
||||
|
||||
let mut events = Vec::new();
|
||||
for row in rows {
|
||||
let event = CalendarEvent::with_id(
|
||||
row.get("id"),
|
||||
row.get("calendar_id"),
|
||||
row.get("summary"),
|
||||
row.get::<Option<String>, _>("description"),
|
||||
row.get::<Option<String>, _>("location"),
|
||||
row.get("start_time"),
|
||||
row.get("end_time"),
|
||||
row.get("all_day"),
|
||||
row.get::<Option<String>, _>("rrule"),
|
||||
row.get("ical_uid"),
|
||||
row.get("ical_data"),
|
||||
row.get("created_at"),
|
||||
row.get("updated_at"),
|
||||
)
|
||||
.map_err(|e| {
|
||||
DomainError::database_error(format!("Error creating calendar event: {}", e))
|
||||
})?;
|
||||
events.push(event);
|
||||
}
|
||||
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
async fn count_events_in_calendar(
|
||||
&self,
|
||||
calendar_id: &Uuid,
|
||||
|
||||
@@ -247,13 +247,44 @@ impl ContactRepository for ContactPgRepository {
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_contacts_by_uids(
|
||||
&self,
|
||||
address_book_id: &Uuid,
|
||||
uids: &[String],
|
||||
) -> ContactRepositoryResult<Vec<Contact>> {
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT
|
||||
id, address_book_id, uid, full_name, first_name, last_name, nickname,
|
||||
email, phone, address, organization, title, notes, photo_url,
|
||||
birthday, anniversary, vcard, etag, created_at, updated_at
|
||||
FROM carddav.contacts
|
||||
WHERE address_book_id = $1 AND uid = ANY($2)
|
||||
ORDER BY full_name, first_name, last_name
|
||||
"#,
|
||||
)
|
||||
.bind(address_book_id)
|
||||
.bind(uids)
|
||||
.fetch_all(&*self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::database_error(format!("Failed to get contacts by uids: {}", e))
|
||||
})?;
|
||||
|
||||
let mut contacts = Vec::new();
|
||||
for row in &rows {
|
||||
contacts.push(Self::row_to_contact(row)?);
|
||||
}
|
||||
Ok(contacts)
|
||||
}
|
||||
|
||||
async fn get_contacts_by_address_book(
|
||||
&self,
|
||||
address_book_id: &Uuid,
|
||||
) -> ContactRepositoryResult<Vec<Contact>> {
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT
|
||||
SELECT
|
||||
id, address_book_id, uid, full_name, first_name, last_name, nickname,
|
||||
email, phone, address, organization, title, notes, photo_url,
|
||||
birthday, anniversary, vcard, etag, created_at, updated_at
|
||||
@@ -276,6 +307,43 @@ impl ContactRepository for ContactPgRepository {
|
||||
Ok(contacts)
|
||||
}
|
||||
|
||||
async fn get_contacts_by_address_book_paginated(
|
||||
&self,
|
||||
address_book_id: &Uuid,
|
||||
limit: i64,
|
||||
offset: i64,
|
||||
) -> ContactRepositoryResult<Vec<Contact>> {
|
||||
let rows = sqlx::query(
|
||||
r#"
|
||||
SELECT
|
||||
id, address_book_id, uid, full_name, first_name, last_name, nickname,
|
||||
email, phone, address, organization, title, notes, photo_url,
|
||||
birthday, anniversary, vcard, etag, created_at, updated_at
|
||||
FROM carddav.contacts
|
||||
WHERE address_book_id = $1
|
||||
ORDER BY full_name, first_name, last_name
|
||||
LIMIT $2 OFFSET $3
|
||||
"#,
|
||||
)
|
||||
.bind(address_book_id)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(&*self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::database_error(format!(
|
||||
"Failed to get contacts by address book (paginated): {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
let mut contacts = Vec::new();
|
||||
for row in &rows {
|
||||
contacts.push(Self::row_to_contact(row)?);
|
||||
}
|
||||
Ok(contacts)
|
||||
}
|
||||
|
||||
async fn get_contacts_by_email(&self, email: &str) -> ContactRepositoryResult<Vec<Contact>> {
|
||||
let search_pattern = super::like_escape(email);
|
||||
|
||||
|
||||
@@ -1515,6 +1515,401 @@ impl DedupService {
|
||||
|
||||
Ok((total_deleted, total_bytes))
|
||||
}
|
||||
|
||||
// ── Legacy whole-file blob re-chunk migration ────────────────
|
||||
//
|
||||
// Files uploaded before CDC chunking landed (migration
|
||||
// 20260414000000_chunk_manifests) are stored as ONE whole-file blob with
|
||||
// no manifest. Every legacy fallback in this service exists to serve
|
||||
// them — and with encryption enabled, a Range read of one decrypts the
|
||||
// ENTIRE blob (AES-GCM is all-or-nothing per blob).
|
||||
//
|
||||
// This migration converts each legacy blob into a regular CDC file:
|
||||
// after it, the converted file is indistinguishable from a native CDC
|
||||
// upload, every read takes the chunked path, and the legacy fallbacks
|
||||
// go permanently cold (they remain as the safety net while a deployment
|
||||
// is mid-migration; they can be deleted from the codebase once fleets
|
||||
// report `legacy re-chunk: nothing to do`).
|
||||
//
|
||||
// Per-hash algorithm:
|
||||
// 1. Spool the blob to a temp file via the normal read path (this
|
||||
// decrypts it when encryption is on), verifying BLAKE3 == hash.
|
||||
// 2. CDC-chunk the spool + store chunks (`store_chunks` bumps each
|
||||
// distinct chunk once — the manifest's reference).
|
||||
// 3. One short accounting TX with the blob row locked:
|
||||
// manifest INSERT with ref_count = N (current file rows referencing
|
||||
// the hash), blob ref_count -= N (those references now live on the
|
||||
// manifest), DELETE the blob row only if it hits exactly 0.
|
||||
// 4. Physically delete the whole-file blob only when its row was
|
||||
// removed. Single-chunk files (chunk hash == file hash) keep the
|
||||
// physical blob — it IS the chunk; only the bookkeeping moves.
|
||||
//
|
||||
// Concurrency: the row lock serializes against the file-delete trigger
|
||||
// and the legacy dedup-hit path. A racing identical upload can land one
|
||||
// legacy reference after our commit; the blob row then survives (> 0)
|
||||
// and that file stays readable through the legacy fallback — a bounded
|
||||
// space leak, never data loss. A crash between step 2 and 3 leaks one
|
||||
// +1 on that file's chunk refs (re-run re-bumps); also a bounded leak,
|
||||
// never data loss.
|
||||
|
||||
/// Count legacy whole-file blobs still referenced by at least one file
|
||||
/// row (the migration's work queue). Runs on the maintenance pool.
|
||||
pub async fn count_legacy_blobs(&self) -> Result<i64, DomainError> {
|
||||
sqlx::query_scalar(
|
||||
"SELECT COUNT(*) FROM storage.blobs b
|
||||
WHERE NOT EXISTS (SELECT 1 FROM storage.chunk_manifests m
|
||||
WHERE m.file_hash = b.hash)
|
||||
AND EXISTS (SELECT 1 FROM storage.files f
|
||||
WHERE f.blob_hash = b.hash)",
|
||||
)
|
||||
.fetch_one(self.maintenance_pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Count legacy blobs: {e}")))
|
||||
}
|
||||
|
||||
/// Spawn the legacy re-chunk migration as a background task.
|
||||
///
|
||||
/// Zero-cost when no legacy blobs exist (one COUNT query, debug log).
|
||||
/// Called from the composition root after `initialize()`.
|
||||
pub fn spawn_legacy_rechunk(self: &Arc<Self>) {
|
||||
let svc = Arc::clone(self);
|
||||
tokio::spawn(async move {
|
||||
match svc.count_legacy_blobs().await {
|
||||
Ok(0) => {
|
||||
tracing::debug!("Legacy re-chunk: no legacy whole-file blobs — nothing to do");
|
||||
}
|
||||
Ok(n) => {
|
||||
tracing::info!(
|
||||
"Legacy re-chunk: {n} pre-CDC whole-file blob(s) referenced by files — \
|
||||
starting background migration (maintenance pool)"
|
||||
);
|
||||
match svc.rechunk_legacy_blobs().await {
|
||||
Ok(report) => tracing::info!(
|
||||
migrated = report.migrated,
|
||||
failed = report.failed,
|
||||
freed_bytes = report.freed_bytes,
|
||||
"Legacy re-chunk complete: {} blob(s) converted to CDC manifests, \
|
||||
{} failed (left untouched), {} bytes of whole-file blobs freed",
|
||||
report.migrated,
|
||||
report.failed,
|
||||
report.freed_bytes,
|
||||
),
|
||||
Err(e) => tracing::error!("Legacy re-chunk aborted: {e}"),
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Legacy re-chunk: startup count failed: {e}"),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Convert every legacy whole-file blob into CDC chunks + manifest.
|
||||
///
|
||||
/// Incremental and resumable: a manifest row is the per-hash "done"
|
||||
/// marker, so re-running after a crash continues where it left off.
|
||||
/// Per-hash failures (e.g. a corrupt blob that no longer matches its
|
||||
/// hash) are logged, counted, and skipped — they never block the sweep.
|
||||
pub async fn rechunk_legacy_blobs(&self) -> Result<LegacyRechunkReport, DomainError> {
|
||||
const BATCH_SIZE: i64 = 64;
|
||||
/// Hard cap on per-hash failures before aborting the sweep — if
|
||||
/// this many blobs are corrupt something is systemically wrong and
|
||||
/// an operator should look before we touch anything else.
|
||||
const MAX_FAILURES: usize = 1_000;
|
||||
|
||||
let mut report = LegacyRechunkReport::default();
|
||||
// Failed hashes are excluded from the candidate query so a corrupt
|
||||
// blob cannot make the sweep loop forever.
|
||||
let mut failed_hashes: Vec<String> = Vec::new();
|
||||
|
||||
loop {
|
||||
let batch: Vec<(String, Option<String>)> = sqlx::query_as(
|
||||
"SELECT b.hash, b.content_type FROM storage.blobs b
|
||||
WHERE NOT EXISTS (SELECT 1 FROM storage.chunk_manifests m
|
||||
WHERE m.file_hash = b.hash)
|
||||
AND EXISTS (SELECT 1 FROM storage.files f
|
||||
WHERE f.blob_hash = b.hash)
|
||||
AND NOT (b.hash = ANY($2))
|
||||
ORDER BY b.hash
|
||||
LIMIT $1",
|
||||
)
|
||||
.bind(BATCH_SIZE)
|
||||
.bind(&failed_hashes)
|
||||
.fetch_all(self.maintenance_pool.as_ref())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::internal_error("Dedup", format!("Legacy candidate query: {e}"))
|
||||
})?;
|
||||
|
||||
if batch.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
for (hash, content_type) in batch {
|
||||
match self.rechunk_one_legacy_blob(&hash, content_type).await {
|
||||
Ok(freed) => {
|
||||
report.migrated += 1;
|
||||
report.freed_bytes += freed;
|
||||
if report.migrated % 50 == 0 {
|
||||
tracing::info!(
|
||||
"Legacy re-chunk progress: {} migrated, {} failed",
|
||||
report.migrated,
|
||||
report.failed
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
report.failed += 1;
|
||||
tracing::error!(
|
||||
"Legacy re-chunk: blob {} failed (left untouched): {e}",
|
||||
&hash[..hash.len().min(12)],
|
||||
);
|
||||
failed_hashes.push(hash);
|
||||
if failed_hashes.len() >= MAX_FAILURES {
|
||||
return Err(DomainError::internal_error(
|
||||
"Dedup",
|
||||
format!(
|
||||
"Legacy re-chunk: aborting after {MAX_FAILURES} per-blob \
|
||||
failures — inspect blob storage integrity"
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// Migrate a single legacy whole-file blob. Returns the number of
|
||||
/// physical bytes freed (0 when the blob doubles as its own chunk).
|
||||
async fn rechunk_one_legacy_blob(
|
||||
&self,
|
||||
hash: &str,
|
||||
content_type: Option<String>,
|
||||
) -> Result<u64, DomainError> {
|
||||
// ── 1. Spool + verify (decrypts via the normal read path) ──
|
||||
// The spooled, hash-verified plaintext is the source of truth for
|
||||
// sizes — `storage.blobs.size` is legacy metadata we don't trust
|
||||
// for the manifest's Range arithmetic.
|
||||
//
|
||||
// The path carries a per-attempt UUID: two processes sharing a temp
|
||||
// dir and racing on the same hash must never truncate or delete each
|
||||
// other's in-flight spool.
|
||||
let spool = std::env::temp_dir().join(format!(
|
||||
"oxicloud-rechunk-{}-{}.tmp",
|
||||
&hash[..hash.len().min(16)],
|
||||
uuid::Uuid::new_v4()
|
||||
));
|
||||
let result = self.spool_and_chunk(hash, &spool).await;
|
||||
let _ = fs::remove_file(&spool).await;
|
||||
let (chunk_hashes, chunk_sizes) = result?;
|
||||
let total_size: u64 = chunk_sizes.iter().sum();
|
||||
|
||||
// ── 2. Accounting TX: move the file references onto the manifest ──
|
||||
let mut tx =
|
||||
self.maintenance_pool.begin().await.map_err(|e| {
|
||||
DomainError::internal_error("Dedup", format!("Rechunk TX begin: {e}"))
|
||||
})?;
|
||||
|
||||
// Lock the legacy blob row — serializes against the file-delete
|
||||
// trigger and the legacy dedup-hit path for this hash.
|
||||
let blob_row_exists = sqlx::query_scalar::<_, i32>(
|
||||
"SELECT ref_count FROM storage.blobs WHERE hash = $1 FOR UPDATE",
|
||||
)
|
||||
.bind(hash)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk lock blob: {e}")))?
|
||||
.is_some();
|
||||
|
||||
let file_refs: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM storage.files WHERE blob_hash = $1")
|
||||
.bind(hash)
|
||||
.fetch_one(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::internal_error("Dedup", format!("Rechunk count refs: {e}"))
|
||||
})?;
|
||||
|
||||
// ref_count = N file references; if every reference vanished while
|
||||
// we were spooling, the zero-ref manifest is swept by the existing
|
||||
// GC (which also unwinds the chunk refs taken in store_chunks).
|
||||
let inserted = sqlx::query(
|
||||
"INSERT INTO storage.chunk_manifests
|
||||
(file_hash, chunk_hashes, chunk_sizes, total_size, chunk_count,
|
||||
content_type, ref_count)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (file_hash) DO NOTHING",
|
||||
)
|
||||
.bind(hash)
|
||||
.bind(&chunk_hashes)
|
||||
.bind(chunk_sizes.iter().map(|s| *s as i64).collect::<Vec<_>>())
|
||||
.bind(total_size as i64)
|
||||
.bind(chunk_hashes.len() as i32)
|
||||
.bind(&content_type)
|
||||
.bind(file_refs as i32)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk manifest: {e}")))?
|
||||
.rows_affected();
|
||||
|
||||
if inserted == 0 {
|
||||
// A manifest appeared concurrently — only possible if the same
|
||||
// content was re-uploaded and fully stored during our spool.
|
||||
// Their bookkeeping is already correct; drop ours.
|
||||
tx.rollback().await.ok();
|
||||
self.release_chunk_refs(&chunk_hashes).await;
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// The N file references now live on the manifest; remove them from
|
||||
// the legacy blob and drop its row only when nothing else (other
|
||||
// manifests using this blob as a chunk, racing legacy references)
|
||||
// still points at it.
|
||||
let mut blob_row_deleted = false;
|
||||
if blob_row_exists {
|
||||
sqlx::query(
|
||||
"UPDATE storage.blobs
|
||||
SET ref_count = GREATEST(ref_count - $2, 0)
|
||||
WHERE hash = $1",
|
||||
)
|
||||
.bind(hash)
|
||||
.bind(file_refs as i32)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::internal_error("Dedup", format!("Rechunk deref blob: {e}"))
|
||||
})?;
|
||||
|
||||
blob_row_deleted =
|
||||
sqlx::query("DELETE FROM storage.blobs WHERE hash = $1 AND ref_count = 0")
|
||||
.bind(hash)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
DomainError::internal_error("Dedup", format!("Rechunk drop blob: {e}"))
|
||||
})?
|
||||
.rows_affected()
|
||||
> 0;
|
||||
}
|
||||
|
||||
tx.commit()
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk commit: {e}")))?;
|
||||
|
||||
// ── 3. Physical cleanup (after commit) ──
|
||||
// Deleted row ⇒ the hash is not one of its own chunks (a single-chunk
|
||||
// file keeps ref_count ≥ 1 from the manifest), but guard anyway.
|
||||
let mut freed = 0;
|
||||
if blob_row_deleted && !chunk_hashes.iter().any(|c| c == hash) {
|
||||
match self.backend.delete_blob(hash).await {
|
||||
Ok(()) => freed = total_size,
|
||||
Err(e) => tracing::warn!(
|
||||
"Legacy re-chunk: converted {} but failed to delete the \
|
||||
old whole-file blob (GC will not retry — row is gone): {e}",
|
||||
&hash[..hash.len().min(12)],
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
"Legacy re-chunk: {} → {} chunk(s), {} file ref(s) moved to manifest{}",
|
||||
&hash[..hash.len().min(12)],
|
||||
chunk_hashes.len(),
|
||||
file_refs,
|
||||
if blob_row_deleted {
|
||||
", whole-file blob freed"
|
||||
} else {
|
||||
""
|
||||
},
|
||||
);
|
||||
|
||||
Ok(freed)
|
||||
}
|
||||
|
||||
/// Spool a legacy blob to `spool`, verify its BLAKE3 matches `hash`,
|
||||
/// CDC-chunk it and store the chunks. Returns (chunk_hashes, chunk_sizes).
|
||||
async fn spool_and_chunk(
|
||||
&self,
|
||||
hash: &str,
|
||||
spool: &Path,
|
||||
) -> Result<(Vec<String>, Vec<u64>), DomainError> {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let mut stream = self.read_blob_stream(hash).await?;
|
||||
let file = fs::File::create(spool)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk spool: {e}")))?;
|
||||
let mut writer = tokio::io::BufWriter::with_capacity(512 * 1024, file);
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk read: {e}")))?;
|
||||
hasher.update(&chunk);
|
||||
writer
|
||||
.write_all(&chunk)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk write: {e}")))?;
|
||||
}
|
||||
writer
|
||||
.flush()
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk flush: {e}")))?;
|
||||
|
||||
let actual = hasher.finalize().to_hex().to_string();
|
||||
if actual != hash {
|
||||
return Err(DomainError::internal_error(
|
||||
"Dedup",
|
||||
format!("Blob content does not match its hash (expected {hash}, got {actual})"),
|
||||
));
|
||||
}
|
||||
|
||||
// Empty blobs can't be mmap'd by the CDC analyser; they become an
|
||||
// empty manifest (the chunked read path streams zero chunks).
|
||||
let spooled_len = fs::metadata(spool)
|
||||
.await
|
||||
.map_err(|e| DomainError::internal_error("Dedup", format!("Rechunk stat: {e}")))?
|
||||
.len();
|
||||
if spooled_len == 0 {
|
||||
return Ok((Vec::new(), Vec::new()));
|
||||
}
|
||||
|
||||
let chunks = Self::cdc_chunk_file(spool)
|
||||
.await
|
||||
.map_err(DomainError::from)?;
|
||||
self.store_chunks(spool, &chunks).await
|
||||
}
|
||||
|
||||
/// Best-effort compensation: drop the per-manifest chunk references
|
||||
/// taken by `store_chunks` when the manifest insert was abandoned.
|
||||
async fn release_chunk_refs(&self, chunk_hashes: &[String]) {
|
||||
if chunk_hashes.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Err(e) = sqlx::query(
|
||||
"UPDATE storage.blobs SET ref_count = GREATEST(ref_count - 1, 0)
|
||||
WHERE hash = ANY($1)",
|
||||
)
|
||||
.bind(chunk_hashes)
|
||||
.execute(self.maintenance_pool.as_ref())
|
||||
.await
|
||||
{
|
||||
tracing::warn!("Legacy re-chunk: failed to release chunk refs: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of a [`DedupService::rechunk_legacy_blobs`] sweep.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct LegacyRechunkReport {
|
||||
/// Legacy blobs successfully converted to CDC manifests.
|
||||
pub migrated: u64,
|
||||
/// Blobs that failed (corrupt / unreadable) and were left untouched.
|
||||
pub failed: u64,
|
||||
/// Physical bytes of whole-file blobs deleted after conversion.
|
||||
pub freed_bytes: u64,
|
||||
}
|
||||
|
||||
// ─── Port implementation ─────────────────────────────────────────────────────
|
||||
@@ -1937,3 +2332,355 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Integration tests for the legacy re-chunk migration — require the test
|
||||
// database (run via `just test-integration`, which spawns it and applies
|
||||
// migrations). Gated on `--cfg integration_tests` like the other PG suites.
|
||||
//
|
||||
// Each test seeds its own synthetic "legacy" state (a whole-file blob row in
|
||||
// `storage.blobs` + file rows pointing at it, no manifest) with unique
|
||||
// `rust-test-rechunk-*` names, then runs the sweep and asserts on the DB
|
||||
// state for ITS hash only — concurrent test sweeps may migrate each other's
|
||||
// blobs first, which is fine (and exercises the idempotency paths).
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
#[cfg(integration_tests)]
|
||||
#[allow(dead_code)]
|
||||
mod rechunk_integration_tests {
|
||||
use super::*;
|
||||
use crate::infrastructure::services::encrypted_blob_backend::EncryptedBlobBackend;
|
||||
use crate::infrastructure::services::local_blob_backend::LocalBlobBackend;
|
||||
use crate::integration_test_support::{ensure_clean_test_db, test_db_url};
|
||||
use sqlx::Row;
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use tempfile::TempDir;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn test_pool() -> Arc<PgPool> {
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(4)
|
||||
.connect(&test_db_url())
|
||||
.await
|
||||
.expect("connect to test DB — run tests/common/spawn-db.sh first");
|
||||
ensure_clean_test_db(&pool).await;
|
||||
Arc::new(pool)
|
||||
}
|
||||
|
||||
async fn seed_user(pool: &PgPool) -> Uuid {
|
||||
sqlx::query("SELECT id FROM auth.users LIMIT 1")
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map(|r| r.get::<Uuid, _>("id"))
|
||||
.expect("auth.users must be seeded (init-test-schema.sh)")
|
||||
}
|
||||
|
||||
/// Plain local backend in a fresh temp dir.
|
||||
async fn local_svc(pool: &Arc<PgPool>, dir: &TempDir) -> DedupService {
|
||||
let backend = Arc::new(LocalBlobBackend::new(&dir.path().join("blobs")));
|
||||
backend.initialize().await.expect("init backend");
|
||||
DedupService::new(backend, pool.clone(), pool.clone())
|
||||
}
|
||||
|
||||
/// AES-256-GCM-encrypted local backend in a fresh temp dir.
|
||||
async fn encrypted_svc(pool: &Arc<PgPool>, dir: &TempDir) -> DedupService {
|
||||
let inner = Arc::new(LocalBlobBackend::new(&dir.path().join("blobs")));
|
||||
inner.initialize().await.expect("init backend");
|
||||
let key = EncryptedBlobBackend::generate_key();
|
||||
let backend = Arc::new(EncryptedBlobBackend::new(inner, &key));
|
||||
DedupService::new(backend, pool.clone(), pool.clone())
|
||||
}
|
||||
|
||||
/// Non-trivial content of `len` bytes + a random 16-byte tail, so every
|
||||
/// invocation produces a unique hash — stale rows left behind by a
|
||||
/// previously failed run (panics skip cleanup) can never collide with
|
||||
/// the current one.
|
||||
fn content(len: usize, salt: u8) -> Vec<u8> {
|
||||
let mut data: Vec<u8> = (0..len)
|
||||
.map(|i| {
|
||||
((i % 251) as u8)
|
||||
.wrapping_add(salt)
|
||||
.wrapping_add((i / 7919) as u8)
|
||||
})
|
||||
.collect();
|
||||
data.extend_from_slice(Uuid::new_v4().as_bytes());
|
||||
data
|
||||
}
|
||||
|
||||
/// Seed a pre-CDC legacy blob: physical blob via the backend + a
|
||||
/// `storage.blobs` row (ref_count = n_files) + `n_files` file rows.
|
||||
/// Returns (hash, file row ids). When `corrupt_stored_bytes` is Some,
|
||||
/// the PHYSICAL content differs from the indexed hash.
|
||||
async fn seed_legacy(
|
||||
svc: &DedupService,
|
||||
pool: &PgPool,
|
||||
dir: &TempDir,
|
||||
data: &[u8],
|
||||
n_files: i32,
|
||||
label: &str,
|
||||
corrupt_stored_bytes: Option<&[u8]>,
|
||||
) -> (String, Vec<Uuid>) {
|
||||
let hash = blake3::hash(data).to_hex().to_string();
|
||||
let stored = corrupt_stored_bytes.unwrap_or(data);
|
||||
|
||||
let src = dir.path().join(format!("seed-{label}.tmp"));
|
||||
tokio::fs::write(&src, stored).await.expect("write seed");
|
||||
svc.backend().put_blob(&hash, &src).await.expect("put blob");
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO storage.blobs (hash, size, ref_count, content_type)
|
||||
VALUES ($1, $2, $3, 'application/octet-stream')
|
||||
ON CONFLICT (hash) DO UPDATE SET ref_count = storage.blobs.ref_count + $3",
|
||||
)
|
||||
.bind(&hash)
|
||||
.bind(data.len() as i64)
|
||||
.bind(n_files)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("insert legacy blob row");
|
||||
|
||||
let user_id = seed_user(pool).await;
|
||||
let mut file_ids = Vec::new();
|
||||
for i in 0..n_files {
|
||||
let name = format!(
|
||||
"rust-test-rechunk-{label}-{}-{i}",
|
||||
&Uuid::new_v4().to_string()[..8]
|
||||
);
|
||||
let id: Uuid = sqlx::query_scalar(
|
||||
"INSERT INTO storage.files (name, user_id, blob_hash, size)
|
||||
VALUES ($1, $2, $3, $4) RETURNING id",
|
||||
)
|
||||
.bind(&name)
|
||||
.bind(user_id)
|
||||
.bind(&hash)
|
||||
.bind(data.len() as i64)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.expect("insert file row");
|
||||
file_ids.push(id);
|
||||
}
|
||||
(hash, file_ids)
|
||||
}
|
||||
|
||||
/// Best-effort cleanup of everything a test seeded/created for `hash`.
|
||||
async fn cleanup(pool: &PgPool, hash: &str, file_ids: &[Uuid]) {
|
||||
let chunks: Option<Vec<String>> = sqlx::query_scalar(
|
||||
"SELECT chunk_hashes FROM storage.chunk_manifests WHERE file_hash = $1",
|
||||
)
|
||||
.bind(hash)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
|
||||
let _ = sqlx::query("DELETE FROM storage.files WHERE id = ANY($1)")
|
||||
.bind(file_ids)
|
||||
.execute(pool)
|
||||
.await;
|
||||
// Also scrub test-named rows from previously failed runs (panics
|
||||
// skip the end-of-test cleanup) that reference the same hash.
|
||||
let _ = sqlx::query(
|
||||
"DELETE FROM storage.files
|
||||
WHERE blob_hash = $1 AND name LIKE 'rust-test-rechunk-%'",
|
||||
)
|
||||
.bind(hash)
|
||||
.execute(pool)
|
||||
.await;
|
||||
let _ = sqlx::query("DELETE FROM storage.chunk_manifests WHERE file_hash = $1")
|
||||
.bind(hash)
|
||||
.execute(pool)
|
||||
.await;
|
||||
let mut to_drop = chunks.unwrap_or_default();
|
||||
to_drop.push(hash.to_string());
|
||||
let _ = sqlx::query("DELETE FROM storage.blobs WHERE hash = ANY($1)")
|
||||
.bind(&to_drop)
|
||||
.execute(pool)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn collect(svc: &DedupService, hash: &str) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
let mut stream = svc.read_blob_stream(hash).await.expect("stream");
|
||||
while let Some(chunk) = stream.next().await {
|
||||
out.extend_from_slice(&chunk.expect("chunk"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Manifest row (ref_count, total_size, chunk_hashes), if present.
|
||||
async fn manifest(pool: &PgPool, hash: &str) -> Option<(i32, i64, Vec<String>)> {
|
||||
sqlx::query_as(
|
||||
"SELECT ref_count, total_size, chunk_hashes
|
||||
FROM storage.chunk_manifests WHERE file_hash = $1",
|
||||
)
|
||||
.bind(hash)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.expect("manifest query")
|
||||
}
|
||||
|
||||
async fn blob_row(pool: &PgPool, hash: &str) -> Option<i32> {
|
||||
sqlx::query_scalar("SELECT ref_count FROM storage.blobs WHERE hash = $1")
|
||||
.bind(hash)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.expect("blob query")
|
||||
}
|
||||
|
||||
// ── 1. Multi-chunk blob: refs move to manifest, whole-file blob freed ──
|
||||
#[tokio::test]
|
||||
async fn rechunk_multi_chunk_moves_refs_and_frees_blob() {
|
||||
let pool = test_pool().await;
|
||||
let dir = TempDir::new().unwrap();
|
||||
let svc = local_svc(&pool, &dir).await;
|
||||
|
||||
// 3 MiB ⇒ ≥ 3 CDC chunks (max chunk = 1 MiB), 2 referencing files.
|
||||
let data = content(3 * 1024 * 1024, 1);
|
||||
let (hash, files) = seed_legacy(&svc, &pool, &dir, &data, 2, "multi", None).await;
|
||||
|
||||
assert!(svc.count_legacy_blobs().await.unwrap() >= 1);
|
||||
svc.rechunk_legacy_blobs().await.expect("sweep");
|
||||
|
||||
let (rc, total, chunks) = manifest(&pool, &hash).await.expect("manifest created");
|
||||
assert_eq!(rc, 2, "both file references must move to the manifest");
|
||||
assert_eq!(total, data.len() as i64);
|
||||
assert!(chunks.len() >= 3, "3 MiB must split into ≥3 chunks");
|
||||
|
||||
// Whole-file blob fully dereferenced: row gone, physical file gone.
|
||||
assert_eq!(blob_row(&pool, &hash).await, None);
|
||||
assert!(!svc.backend().blob_exists(&hash).await.unwrap());
|
||||
|
||||
// Every chunk row carries exactly the manifest's reference.
|
||||
for c in &chunks {
|
||||
assert_eq!(blob_row(&pool, c).await, Some(1), "chunk {c}");
|
||||
}
|
||||
|
||||
// Content integrity through the chunked read path + a Range that
|
||||
// crosses a chunk boundary.
|
||||
assert_eq!(collect(&svc, &hash).await, data);
|
||||
let mut ranged = Vec::new();
|
||||
let mut s = svc
|
||||
.read_blob_range_stream(&hash, 1_500_000, Some(1_500_100))
|
||||
.await
|
||||
.expect("range");
|
||||
while let Some(chunk) = s.next().await {
|
||||
ranged.extend_from_slice(&chunk.expect("chunk"));
|
||||
}
|
||||
assert_eq!(ranged, &data[1_500_000..1_500_100]);
|
||||
|
||||
cleanup(&pool, &hash, &files).await;
|
||||
}
|
||||
|
||||
// ── 2. Single-chunk blob: physical blob IS the chunk and must survive ──
|
||||
#[tokio::test]
|
||||
async fn rechunk_single_chunk_keeps_physical_blob() {
|
||||
let pool = test_pool().await;
|
||||
let dir = TempDir::new().unwrap();
|
||||
let svc = local_svc(&pool, &dir).await;
|
||||
|
||||
// 50 KB < CDC_MIN_CHUNK ⇒ exactly one chunk whose hash == file hash.
|
||||
let data = content(50 * 1024, 2);
|
||||
let (hash, files) = seed_legacy(&svc, &pool, &dir, &data, 1, "single", None).await;
|
||||
|
||||
svc.rechunk_legacy_blobs().await.expect("sweep");
|
||||
|
||||
let (rc, total, chunks) = manifest(&pool, &hash).await.expect("manifest created");
|
||||
assert_eq!(rc, 1);
|
||||
assert_eq!(total, data.len() as i64);
|
||||
assert_eq!(chunks, vec![hash.clone()], "the file IS its single chunk");
|
||||
|
||||
// Blob row survives with exactly the manifest's chunk reference;
|
||||
// the physical bytes were never rewritten.
|
||||
assert_eq!(blob_row(&pool, &hash).await, Some(1));
|
||||
assert!(svc.backend().blob_exists(&hash).await.unwrap());
|
||||
assert_eq!(collect(&svc, &hash).await, data);
|
||||
|
||||
cleanup(&pool, &hash, &files).await;
|
||||
}
|
||||
|
||||
// ── 3. Corrupt blob (content ≠ hash): fail, count, leave untouched ──
|
||||
#[tokio::test]
|
||||
async fn rechunk_corrupt_blob_left_untouched() {
|
||||
let pool = test_pool().await;
|
||||
let dir = TempDir::new().unwrap();
|
||||
let svc = local_svc(&pool, &dir).await;
|
||||
|
||||
let data = content(100 * 1024, 3);
|
||||
let mut wrong = data.clone();
|
||||
wrong[0] ^= 0xFF;
|
||||
let (hash, files) = seed_legacy(&svc, &pool, &dir, &data, 1, "corrupt", Some(&wrong)).await;
|
||||
|
||||
let report = svc.rechunk_legacy_blobs().await.expect("sweep");
|
||||
assert!(report.failed >= 1, "the corrupt blob must be counted");
|
||||
|
||||
// Nothing was touched: no manifest, blob row + refs + file intact.
|
||||
assert_eq!(manifest(&pool, &hash).await, None);
|
||||
assert_eq!(blob_row(&pool, &hash).await, Some(1));
|
||||
assert!(svc.backend().blob_exists(&hash).await.unwrap());
|
||||
let files_left: i64 =
|
||||
sqlx::query_scalar("SELECT COUNT(*) FROM storage.files WHERE blob_hash = $1")
|
||||
.bind(&hash)
|
||||
.fetch_one(pool.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(files_left, 1);
|
||||
|
||||
cleanup(&pool, &hash, &files).await;
|
||||
}
|
||||
|
||||
// ── 4. Empty blob: empty manifest, empty stream ──
|
||||
#[tokio::test]
|
||||
async fn rechunk_empty_blob() {
|
||||
let pool = test_pool().await;
|
||||
let dir = TempDir::new().unwrap();
|
||||
let svc = local_svc(&pool, &dir).await;
|
||||
|
||||
// The empty-content hash is a constant (no per-run uniqueness is
|
||||
// possible), so scrub any leftovers from a previously failed run.
|
||||
let empty_hash = blake3::hash(&[]).to_hex().to_string();
|
||||
cleanup(&pool, &empty_hash, &[]).await;
|
||||
|
||||
let (hash, files) = seed_legacy(&svc, &pool, &dir, &[], 1, "empty", None).await;
|
||||
|
||||
svc.rechunk_legacy_blobs().await.expect("sweep");
|
||||
|
||||
let (rc, total, chunks) = manifest(&pool, &hash).await.expect("manifest created");
|
||||
assert_eq!((rc, total), (1, 0));
|
||||
assert!(chunks.is_empty());
|
||||
assert!(collect(&svc, &hash).await.is_empty());
|
||||
|
||||
cleanup(&pool, &hash, &files).await;
|
||||
}
|
||||
|
||||
// ── 5. Encrypted backend: spool decrypts, chunks re-encrypt, Range works ──
|
||||
#[tokio::test]
|
||||
async fn rechunk_encrypted_multi_chunk_roundtrip() {
|
||||
let pool = test_pool().await;
|
||||
let dir = TempDir::new().unwrap();
|
||||
let svc = encrypted_svc(&pool, &dir).await;
|
||||
|
||||
let data = content(2 * 1024 * 1024 + 333, 4);
|
||||
let (hash, files) = seed_legacy(&svc, &pool, &dir, &data, 1, "enc", None).await;
|
||||
|
||||
svc.rechunk_legacy_blobs().await.expect("sweep");
|
||||
|
||||
let (rc, total, chunks) = manifest(&pool, &hash).await.expect("manifest created");
|
||||
assert_eq!(rc, 1);
|
||||
assert_eq!(total, data.len() as i64);
|
||||
assert!(chunks.len() >= 2);
|
||||
assert_eq!(blob_row(&pool, &hash).await, None, "whole-file blob freed");
|
||||
|
||||
// The point of the whole migration: a Range read now decrypts only
|
||||
// the overlapping ≤1 MiB chunks, and returns correct plaintext.
|
||||
assert_eq!(collect(&svc, &hash).await, data);
|
||||
let mut ranged = Vec::new();
|
||||
let mut s = svc
|
||||
.read_blob_range_stream(&hash, 1_100_000, Some(1_100_064))
|
||||
.await
|
||||
.expect("range");
|
||||
while let Some(chunk) = s.next().await {
|
||||
ranged.extend_from_slice(&chunk.expect("chunk"));
|
||||
}
|
||||
assert_eq!(ranged, &data[1_100_000..1_100_064]);
|
||||
|
||||
cleanup(&pool, &hash, &files).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ use chrono::Utc;
|
||||
use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, Validation, decode, encode};
|
||||
use moka::sync::Cache;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
@@ -69,8 +70,9 @@ impl From<JwtClaims> for TokenClaims {
|
||||
///
|
||||
/// The cache uses the **BLAKE3** hash of the raw token string as key (32-byte,
|
||||
/// ~0.1 µs to compute — 20× cheaper than HMAC verification) and stores the
|
||||
/// validated `TokenClaims`. On a cache hit the HMAC step is completely
|
||||
/// skipped.
|
||||
/// validated claims behind an `Arc`. On a cache hit the HMAC step is
|
||||
/// completely skipped and the lookup returns a refcount bump rather than a
|
||||
/// deep clone of the (multi-`String`) `TokenClaims`.
|
||||
///
|
||||
/// **Security properties**:
|
||||
/// - TTL of 30 s bounds the window in which a revoked token remains valid.
|
||||
@@ -84,8 +86,9 @@ pub struct JwtTokenService {
|
||||
access_token_expiry: i64,
|
||||
/// Expiration time for refresh tokens in seconds
|
||||
refresh_token_expiry: i64,
|
||||
/// Validation result cache: blake3(token) → TokenClaims
|
||||
validation_cache: Cache<[u8; 32], TokenClaims>,
|
||||
/// Validation result cache: blake3(token) → Arc<TokenClaims>.
|
||||
/// `Arc` so a cache hit is a refcount bump, not a multi-`String` clone.
|
||||
validation_cache: Cache<[u8; 32], Arc<TokenClaims>>,
|
||||
/// Cache hit counter (for observability / metrics)
|
||||
cache_hits: AtomicU64,
|
||||
/// Cache miss counter
|
||||
@@ -194,7 +197,7 @@ impl TokenServicePort for JwtTokenService {
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_token(&self, token: &str) -> Result<TokenClaims, DomainError> {
|
||||
fn validate_token(&self, token: &str) -> Result<Arc<TokenClaims>, DomainError> {
|
||||
// ── 1. Fast-path: check the validation cache ─────────────
|
||||
let key = Self::token_hash(token);
|
||||
|
||||
@@ -232,14 +235,15 @@ impl TokenServicePort for JwtTokenService {
|
||||
),
|
||||
})?;
|
||||
|
||||
let claims: TokenClaims = token_data.claims.into();
|
||||
let claims = Arc::new(TokenClaims::from(token_data.claims));
|
||||
|
||||
// ── 3. Store in cache for subsequent requests ────────────
|
||||
// Only cache tokens that won't expire within the cache TTL window,
|
||||
// avoiding stale positives right at the boundary.
|
||||
let remaining_secs = claims.exp - Utc::now().timestamp();
|
||||
if remaining_secs > VALIDATION_CACHE_TTL_SECS as i64 {
|
||||
self.validation_cache.insert(key, claims.clone());
|
||||
// Refcount bump — the claims live once behind the `Arc`.
|
||||
self.validation_cache.insert(key, Arc::clone(&claims));
|
||||
}
|
||||
|
||||
Ok(claims)
|
||||
@@ -348,6 +352,31 @@ mod tests {
|
||||
assert_eq!(misses, 1, "Expected 1 cache miss");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_hit_returns_same_arc_not_a_clone() {
|
||||
let service = JwtTokenService::new(
|
||||
"test_secret_key_at_least_32_bytes_long".to_string(),
|
||||
3600,
|
||||
86400,
|
||||
);
|
||||
let token = service
|
||||
.generate_access_token(&create_test_user())
|
||||
.expect("Should generate token");
|
||||
|
||||
// Miss populates the cache; hit must hand back the very same
|
||||
// allocation (pointer-equal Arc), proving the hot path is a refcount
|
||||
// bump rather than a deep clone of the claims' Strings.
|
||||
let first = service.validate_token(&token).expect("miss");
|
||||
let second = service.validate_token(&token).expect("hit");
|
||||
|
||||
assert!(
|
||||
Arc::ptr_eq(&first, &second),
|
||||
"cache hit must return the same Arc, not a fresh allocation"
|
||||
);
|
||||
let (hits, misses) = service.cache_stats();
|
||||
assert_eq!((hits, misses), (1, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_token_not_cached() {
|
||||
let service = JwtTokenService::new("secret".to_string(), 3600, 86400);
|
||||
|
||||
@@ -1,15 +1,23 @@
|
||||
//! WebDAV lock store backed by Moka (in-memory cache with per-entry TTL).
|
||||
//!
|
||||
//! Locks are automatically evicted when their timeout expires, preventing
|
||||
//! orphaned locks from accumulating. Two caches are maintained:
|
||||
//! Each lock expires automatically at its own RFC 4918 `Timeout`, enforced
|
||||
//! by Moka's [`Expiry`](moka::Expiry) policy. There are **no background
|
||||
//! tasks and no per-lock timers** — Office clients refresh locks
|
||||
//! constantly, and spawning a `sleep` future per acquire/refresh used to
|
||||
//! leave thousands of orphaned timers pinned in the runtime. Two caches are
|
||||
//! maintained:
|
||||
//!
|
||||
//! - `by_path` : path → `LockEntry` (for LOCK conflict detection)
|
||||
//! - `by_token` : token → path (for fast UNLOCK / refresh lookups)
|
||||
//! - `by_path` : path → `LockEntry` (source of truth; precise per-lock TTL)
|
||||
//! - `by_token` : token → path (reverse index for UNLOCK / refresh)
|
||||
//!
|
||||
//! Both caches share the same TTL so entries disappear together.
|
||||
//! `by_path` carries the exact per-lock TTL via `Expiry`; `by_token` keeps a
|
||||
//! 24 h backstop TTL. A reverse-index entry that outlives its lock is
|
||||
//! harmless: every lookup resolves through `by_path`, which is
|
||||
//! authoritative, so an expired lock reads as absent even before its token
|
||||
//! mapping is evicted.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::application::adapters::webdav_adapter::{LockInfo, LockScope};
|
||||
|
||||
@@ -26,6 +34,40 @@ pub struct LockEntry {
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
/// Per-entry expiration policy for the `by_path` cache.
|
||||
///
|
||||
/// Moka calls this on insert (create) and re-insert (update, i.e. refresh)
|
||||
/// to derive each lock's TTL from its own `Timeout` header — replacing the
|
||||
/// old "global TTL + one spawned timer per lock" scheme. Reads do not
|
||||
/// extend the lock (the default `expire_after_read` leaves the remaining
|
||||
/// duration untouched).
|
||||
struct LockExpiry;
|
||||
|
||||
impl moka::Expiry<String, LockEntry> for LockExpiry {
|
||||
fn expire_after_create(
|
||||
&self,
|
||||
_path: &String,
|
||||
entry: &LockEntry,
|
||||
_created_at: Instant,
|
||||
) -> Option<Duration> {
|
||||
Some(WebDavLockStore::parse_timeout(
|
||||
entry.info.timeout.as_deref(),
|
||||
))
|
||||
}
|
||||
|
||||
fn expire_after_update(
|
||||
&self,
|
||||
_path: &String,
|
||||
entry: &LockEntry,
|
||||
_updated_at: Instant,
|
||||
_remaining: Option<Duration>,
|
||||
) -> Option<Duration> {
|
||||
Some(WebDavLockStore::parse_timeout(
|
||||
entry.info.timeout.as_deref(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// In-memory WebDAV lock store with automatic TTL-based expiration.
|
||||
///
|
||||
/// Uses Moka's `sync::Cache` — lock-free (sharded) reads, bounded size,
|
||||
@@ -42,13 +84,16 @@ impl WebDavLockStore {
|
||||
///
|
||||
/// * `max_capacity` — upper bound on simultaneous locks (evicts LRU on overflow).
|
||||
pub fn new(max_capacity: u64) -> Self {
|
||||
// We use `expire_after` (per-entry TTL) via insert with explicit ttl,
|
||||
// so we configure a generous global time_to_live as a safety net.
|
||||
// `by_path` is the source of truth: each lock expires at its own
|
||||
// `Timeout` via the `LockExpiry` policy (no spawned timers).
|
||||
let by_path = moka::sync::Cache::builder()
|
||||
.max_capacity(max_capacity)
|
||||
.time_to_live(Duration::from_secs(MAX_LOCK_TIMEOUT_SECS))
|
||||
.expire_after(LockExpiry)
|
||||
.build();
|
||||
|
||||
// `by_token` is a reverse index; a 24 h backstop TTL bounds any
|
||||
// mapping that outlives its lock. Lookups resolve through `by_path`,
|
||||
// so a lingering entry here never resurrects an expired lock.
|
||||
let by_token = moka::sync::Cache::builder()
|
||||
.max_capacity(max_capacity)
|
||||
.time_to_live(Duration::from_secs(MAX_LOCK_TIMEOUT_SECS))
|
||||
@@ -72,38 +117,17 @@ impl WebDavLockStore {
|
||||
return Err(existing);
|
||||
}
|
||||
|
||||
let ttl = Self::parse_timeout(info.timeout.as_deref());
|
||||
let entry = LockEntry {
|
||||
info,
|
||||
path: path.to_owned(),
|
||||
};
|
||||
|
||||
// `LockExpiry` derives the TTL from `entry.info.timeout` on insert —
|
||||
// no spawned timer needed.
|
||||
self.by_path.insert(path.to_owned(), entry.clone());
|
||||
self.by_token
|
||||
.insert(entry.info.token.clone(), path.to_owned());
|
||||
|
||||
// Moka 0.12 does not expose per-entry set_expiration_after_insert at
|
||||
// insert time. We rely on the global `time_to_live` as an upper bound
|
||||
// and use the `invalidate_after` helper below for custom TTL.
|
||||
//
|
||||
// To implement shorter-than-max TTL we schedule an async invalidation.
|
||||
if ttl.as_secs() < MAX_LOCK_TIMEOUT_SECS {
|
||||
let by_path = self.by_path.clone();
|
||||
let by_token = self.by_token.clone();
|
||||
let token = entry.info.token.clone();
|
||||
let path_owned = path.to_owned();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(ttl).await;
|
||||
// Only remove if the entry still matches (wasn't refreshed/replaced)
|
||||
if let Some(e) = by_path.get(&path_owned)
|
||||
&& e.info.token == token
|
||||
{
|
||||
by_path.invalidate(&path_owned);
|
||||
by_token.invalidate(&token);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
@@ -120,29 +144,13 @@ impl WebDavLockStore {
|
||||
}
|
||||
|
||||
let ttl = Self::parse_timeout(new_timeout.or(entry.info.timeout.as_deref()));
|
||||
let timeout_str = format!("Second-{}", ttl.as_secs());
|
||||
entry.info.timeout = Some(timeout_str.clone());
|
||||
// Normalize the stored timeout so `LockExpiry` recomputes the new TTL
|
||||
// from it on re-insert (Moka fires `expire_after_update`).
|
||||
entry.info.timeout = Some(format!("Second-{}", ttl.as_secs()));
|
||||
|
||||
// Re-insert to reset the TTL
|
||||
self.by_path.insert(path.clone(), entry.clone());
|
||||
self.by_token.insert(token.to_owned(), path.clone());
|
||||
|
||||
if ttl.as_secs() < MAX_LOCK_TIMEOUT_SECS {
|
||||
let by_path = self.by_path.clone();
|
||||
let by_token = self.by_token.clone();
|
||||
let token_owned = token.to_owned();
|
||||
let path_owned = path.clone();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(ttl).await;
|
||||
if let Some(e) = by_path.get(&path_owned)
|
||||
&& e.info.token == token_owned
|
||||
{
|
||||
by_path.invalidate(&path_owned);
|
||||
by_token.invalidate(&token_owned);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Some(entry)
|
||||
}
|
||||
|
||||
@@ -212,3 +220,136 @@ pub fn create_webdav_lock_store() -> Arc<WebDavLockStore> {
|
||||
// if the cap is reached, so stale entries are cleaned automatically.
|
||||
Arc::new(WebDavLockStore::new(10_000))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::application::adapters::webdav_adapter::LockType;
|
||||
use moka::Expiry;
|
||||
|
||||
fn lock_info(token: &str, timeout: Option<&str>, scope: LockScope) -> LockInfo {
|
||||
LockInfo {
|
||||
token: token.to_owned(),
|
||||
owner: Some("tester".to_owned()),
|
||||
depth: "0".to_owned(),
|
||||
timeout: timeout.map(str::to_owned),
|
||||
scope,
|
||||
type_: LockType::Write,
|
||||
}
|
||||
}
|
||||
|
||||
fn entry(token: &str, timeout: Option<&str>) -> LockEntry {
|
||||
LockEntry {
|
||||
info: lock_info(token, timeout, LockScope::Exclusive),
|
||||
path: "/file.txt".to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expiry_uses_per_entry_timeout() {
|
||||
let now = Instant::now();
|
||||
let key = "/file.txt".to_owned();
|
||||
|
||||
// Explicit Second-NNN → that exact duration.
|
||||
let e = entry("t", Some("Second-300"));
|
||||
assert_eq!(
|
||||
LockExpiry.expire_after_create(&key, &e, now),
|
||||
Some(Duration::from_secs(300))
|
||||
);
|
||||
// Refresh path (update) recomputes from the (normalized) timeout.
|
||||
assert_eq!(
|
||||
LockExpiry.expire_after_update(&key, &e, now, None),
|
||||
Some(Duration::from_secs(300))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expiry_clamps_infinite_and_defaults_none() {
|
||||
let now = Instant::now();
|
||||
let key = "/file.txt".to_owned();
|
||||
|
||||
let infinite = entry("t", Some("Infinite"));
|
||||
assert_eq!(
|
||||
LockExpiry.expire_after_create(&key, &infinite, now),
|
||||
Some(Duration::from_secs(MAX_LOCK_TIMEOUT_SECS))
|
||||
);
|
||||
|
||||
let none = entry("t", None);
|
||||
assert_eq!(
|
||||
LockExpiry.expire_after_create(&key, &none, now),
|
||||
Some(Duration::from_secs(DEFAULT_LOCK_TIMEOUT_SECS))
|
||||
);
|
||||
|
||||
// Over-large requests are clamped to the maximum.
|
||||
let huge = entry("t", Some("Second-999999999"));
|
||||
assert_eq!(
|
||||
LockExpiry.expire_after_create(&key, &huge, now),
|
||||
Some(Duration::from_secs(MAX_LOCK_TIMEOUT_SECS))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquire_get_release_roundtrip() {
|
||||
let store = WebDavLockStore::new(16);
|
||||
let info = lock_info("urn:token-1", Some("Second-600"), LockScope::Exclusive);
|
||||
|
||||
let acquired = store.acquire("/a.txt", info).expect("acquire");
|
||||
assert_eq!(acquired.info.token, "urn:token-1");
|
||||
|
||||
// Resolvable by both indexes.
|
||||
assert_eq!(
|
||||
store.get_by_path("/a.txt").map(|e| e.info.token.clone()),
|
||||
Some("urn:token-1".to_owned())
|
||||
);
|
||||
assert_eq!(
|
||||
store.get_by_token("urn:token-1").map(|e| e.path.clone()),
|
||||
Some("/a.txt".to_owned())
|
||||
);
|
||||
|
||||
assert!(store.release("urn:token-1"));
|
||||
assert!(store.get_by_path("/a.txt").is_none());
|
||||
assert!(store.get_by_token("urn:token-1").is_none());
|
||||
// Releasing an unknown token reports nothing removed.
|
||||
assert!(!store.release("urn:token-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exclusive_lock_conflicts() {
|
||||
let store = WebDavLockStore::new(16);
|
||||
store
|
||||
.acquire(
|
||||
"/a.txt",
|
||||
lock_info("urn:token-1", Some("Second-600"), LockScope::Exclusive),
|
||||
)
|
||||
.expect("first acquire");
|
||||
|
||||
let conflict = store.acquire(
|
||||
"/a.txt",
|
||||
lock_info("urn:token-2", Some("Second-600"), LockScope::Exclusive),
|
||||
);
|
||||
assert!(conflict.is_err());
|
||||
// The original holder is returned so the caller can report it.
|
||||
assert_eq!(conflict.unwrap_err().info.token, "urn:token-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_normalizes_timeout_and_keeps_lock() {
|
||||
let store = WebDavLockStore::new(16);
|
||||
store
|
||||
.acquire(
|
||||
"/a.txt",
|
||||
lock_info("urn:token-1", Some("Infinite"), LockScope::Exclusive),
|
||||
)
|
||||
.expect("acquire");
|
||||
|
||||
let refreshed = store
|
||||
.refresh("urn:token-1", Some("Second-120"))
|
||||
.expect("refresh");
|
||||
assert_eq!(refreshed.info.timeout.as_deref(), Some("Second-120"));
|
||||
// Still present and still addressable by token.
|
||||
assert!(store.get_by_token("urn:token-1").is_some());
|
||||
|
||||
// Refreshing an unknown token yields None.
|
||||
assert!(store.refresh("urn:unknown", Some("Second-120")).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ use std::fmt::Write;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::application::adapters::caldav_adapter::{CalDavAdapter, CalDavReportType};
|
||||
use crate::application::adapters::uid_from_multiget_href;
|
||||
use crate::application::adapters::webdav_adapter::{PropFindRequest, PropFindType};
|
||||
use crate::application::dtos::calendar_dto::{
|
||||
CreateCalendarDto, CreateEventICalDto, UpdateCalendarDto,
|
||||
@@ -422,17 +423,13 @@ async fn handle_propfind(
|
||||
}
|
||||
};
|
||||
|
||||
// Individual event .ics
|
||||
// Individual event .ics — indexed lookup by iCalendar UID.
|
||||
let ical_uid = event_path.trim_end_matches(".ics");
|
||||
|
||||
let events = calendar_service
|
||||
.list_events(calendar_id, None, None, user.id)
|
||||
let event = calendar_service
|
||||
.get_event_by_ical_uid(calendar_id, ical_uid, user.id)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to list events: {}", e)))?;
|
||||
|
||||
let event = events
|
||||
.iter()
|
||||
.find(|e| e.ical_uid == ical_uid)
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to look up event: {}", e)))?
|
||||
.ok_or_else(|| AppError::not_found(format!("Event not found: {}", ical_uid)))?;
|
||||
|
||||
let base_href = &format!("/caldav/{}/", calendar_id);
|
||||
@@ -444,7 +441,7 @@ async fn handle_propfind(
|
||||
let mut response_body = Vec::new();
|
||||
CalDavAdapter::generate_calendar_events_response(
|
||||
&mut response_body,
|
||||
std::slice::from_ref(event),
|
||||
std::slice::from_ref(&event),
|
||||
&report_type,
|
||||
base_href,
|
||||
)
|
||||
@@ -502,15 +499,18 @@ async fn handle_report(
|
||||
}
|
||||
}
|
||||
CalDavReportType::CalendarMultiget { hrefs, .. } => {
|
||||
let all_events = calendar_service
|
||||
.list_events(calendar_id, None, None, user.id)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to list events: {}", e)))?;
|
||||
// Indexed batch lookup (`ical_uid = ANY(...)`) — a multiget for
|
||||
// a handful of events must not pay for listing the whole
|
||||
// calendar and filtering client-side.
|
||||
let uids: Vec<String> = hrefs
|
||||
.iter()
|
||||
.filter_map(|href| uid_from_multiget_href(href, ".ics"))
|
||||
.collect();
|
||||
|
||||
all_events
|
||||
.into_iter()
|
||||
.filter(|evt| hrefs.iter().any(|href| href.contains(&evt.ical_uid)))
|
||||
.collect()
|
||||
calendar_service
|
||||
.get_events_by_ical_uids(calendar_id, &uids, user.id)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to fetch events: {}", e)))?
|
||||
}
|
||||
CalDavReportType::SyncCollection { .. } => calendar_service
|
||||
.list_events(calendar_id, None, None, user.id)
|
||||
|
||||
@@ -28,6 +28,7 @@ use std::sync::Arc;
|
||||
use crate::application::adapters::carddav_adapter::{
|
||||
CardDavAdapter, CardDavReportType, contact_to_vcard,
|
||||
};
|
||||
use crate::application::adapters::uid_from_multiget_href;
|
||||
use crate::application::adapters::webdav_adapter::{PropFindRequest, PropFindType};
|
||||
use crate::application::dtos::address_book_dto::{CreateAddressBookDto, UpdateAddressBookDto};
|
||||
use crate::application::dtos::contact_dto::CreateContactVCardDto;
|
||||
@@ -269,7 +270,7 @@ async fn handle_propfind(
|
||||
|
||||
let contacts = if depth != "0" {
|
||||
contact_svc
|
||||
.list_contacts(address_book_id, user.id)
|
||||
.list_contacts(address_book_id, None, None, user.id)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
@@ -359,22 +360,25 @@ async fn handle_report(
|
||||
|
||||
let contacts = match &report {
|
||||
CardDavReportType::AddressbookQuery { .. } => contact_svc
|
||||
.list_contacts(address_book_id, user.id)
|
||||
.list_contacts(address_book_id, None, None, user.id)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to list contacts: {}", e)))?,
|
||||
CardDavReportType::AddressbookMultiget { hrefs, .. } => {
|
||||
let all_contacts = contact_svc
|
||||
.list_contacts(address_book_id, user.id)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to list contacts: {}", e)))?;
|
||||
// Indexed batch lookup (`uid = ANY(...)`) — a multiget for a
|
||||
// handful of contacts must not pay for listing the whole
|
||||
// address book and filtering client-side.
|
||||
let uids: Vec<String> = hrefs
|
||||
.iter()
|
||||
.filter_map(|href| uid_from_multiget_href(href, ".vcf"))
|
||||
.collect();
|
||||
|
||||
all_contacts
|
||||
.into_iter()
|
||||
.filter(|c| hrefs.iter().any(|href| href.contains(&c.uid)))
|
||||
.collect()
|
||||
contact_svc
|
||||
.get_contacts_by_uids(address_book_id, &uids, user.id)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to fetch contacts: {}", e)))?
|
||||
}
|
||||
CardDavReportType::SyncCollection { .. } => contact_svc
|
||||
.list_contacts(address_book_id, user.id)
|
||||
.list_contacts(address_book_id, None, None, user.id)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to list contacts: {}", e)))?,
|
||||
};
|
||||
@@ -559,7 +563,7 @@ async fn handle_get(
|
||||
if parts.len() < 2 {
|
||||
// GET on address book collection — return all contacts as vcf
|
||||
let contacts = contact_svc
|
||||
.list_contacts(address_book_id, user.id)
|
||||
.list_contacts(address_book_id, None, None, user.id)
|
||||
.await
|
||||
.map_err(|e| AppError::internal_error(format!("Failed to list contacts: {}", e)))?;
|
||||
|
||||
|
||||
@@ -119,16 +119,14 @@ pub struct AddMemberRequest {
|
||||
}
|
||||
|
||||
/// Query parameters for paginated listing.
|
||||
///
|
||||
/// Both fields are optional: when omitted, regular address books return
|
||||
/// the full contact list (the frontend relies on this), while the system
|
||||
/// book falls back to its own defaults (limit 100, offset 0).
|
||||
#[derive(Deserialize)]
|
||||
pub struct ListQuery {
|
||||
#[serde(default = "default_limit")]
|
||||
limit: i64,
|
||||
#[serde(default)]
|
||||
offset: i64,
|
||||
}
|
||||
|
||||
fn default_limit() -> i64 {
|
||||
100
|
||||
limit: Option<i64>,
|
||||
offset: Option<i64>,
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
@@ -455,8 +453,8 @@ pub async fn delete_address_book(
|
||||
path = "/api/address-books/{book_id}/contacts",
|
||||
params(
|
||||
("book_id" = String, Path, description = "Address book UUID or \"system\""),
|
||||
("limit" = Option<i64>, Query, description = "Max results (default 100)"),
|
||||
("offset" = Option<i64>, Query, description = "Pagination offset (default 0)"),
|
||||
("limit" = Option<i64>, Query, description = "Max results (omit for the full book; system book defaults to 100)"),
|
||||
("offset" = Option<i64>, Query, description = "Pagination offset (omit for none; system book defaults to 0)"),
|
||||
),
|
||||
responses(
|
||||
(status = 200, description = "List of contacts"),
|
||||
@@ -488,7 +486,10 @@ pub async fn list_contacts(
|
||||
return e.into_response();
|
||||
}
|
||||
let caller_id = auth_user.id.to_string();
|
||||
match auth_service.list_users(params.limit, params.offset).await {
|
||||
match auth_service
|
||||
.list_users(params.limit.unwrap_or(100), params.offset.unwrap_or(0))
|
||||
.await
|
||||
{
|
||||
Ok(users) => {
|
||||
let contacts: Vec<ContactDto> = users
|
||||
.into_iter()
|
||||
@@ -505,7 +506,7 @@ pub async fn list_contacts(
|
||||
} else {
|
||||
match state
|
||||
.contact_service
|
||||
.list_contacts(&book_id, auth_user.id)
|
||||
.list_contacts(&book_id, params.limit, params.offset, auth_user.id)
|
||||
.await
|
||||
{
|
||||
Ok(contacts) => (StatusCode::OK, Json(contacts)).into_response(),
|
||||
|
||||
@@ -54,7 +54,7 @@ pub async fn require_admin(
|
||||
Ok((
|
||||
Uuid::parse_str(&claims.sub)
|
||||
.map_err(|_| AppError::internal_error("Invalid user ID in token"))?,
|
||||
claims.role,
|
||||
claims.role.clone(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -87,6 +87,6 @@ pub async fn require_authenticated(
|
||||
Ok((
|
||||
Uuid::parse_str(&claims.sub)
|
||||
.map_err(|_| AppError::internal_error("Invalid user ID in token"))?,
|
||||
claims.role,
|
||||
claims.role.clone(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -183,9 +183,9 @@ pub async fn auth_middleware(
|
||||
})?;
|
||||
let current_user = Arc::new(CurrentUser {
|
||||
id: user_id,
|
||||
username: claims.username,
|
||||
email: claims.email,
|
||||
role: claims.role,
|
||||
username: claims.username.clone(),
|
||||
email: claims.email.clone(),
|
||||
role: claims.role.clone(),
|
||||
});
|
||||
request.extensions_mut().insert(current_user);
|
||||
tracing::Span::current().record("user_id", user_id.to_string());
|
||||
@@ -287,9 +287,9 @@ pub async fn auth_middleware(
|
||||
})?;
|
||||
let current_user = Arc::new(CurrentUser {
|
||||
id: user_id,
|
||||
username: claims.username,
|
||||
email: claims.email,
|
||||
role: claims.role,
|
||||
username: claims.username.clone(),
|
||||
email: claims.email.clone(),
|
||||
role: claims.role.clone(),
|
||||
});
|
||||
request.extensions_mut().insert(current_user);
|
||||
request.extensions_mut().insert(CookieAuthenticated);
|
||||
|
||||
@@ -36,6 +36,13 @@ function buildResourceIcon(item, resourceType) {
|
||||
|
||||
const canThumbnail = thumbnail?.canHandle(file) ?? false;
|
||||
if (canThumbnail) {
|
||||
// A PDF just entered the list: warm up the pdf.js stack (~1.3 MB)
|
||||
// in the background now, so a thumbnail cache-miss below doesn't
|
||||
// stall its first render on the library download. Idempotent.
|
||||
if (file.mime_type === 'application/pdf') {
|
||||
thumbnail.preloadPdf();
|
||||
}
|
||||
|
||||
const img = document.createElement('img');
|
||||
img.className = 'file-thumb';
|
||||
img.src = `/api/files/${file.id}/thumbnail/icon`;
|
||||
|
||||
@@ -140,6 +140,18 @@ export class ResourceListComponent {
|
||||
*/
|
||||
this._lastGroupEl = null;
|
||||
|
||||
/**
|
||||
* Live swimlane wrappers currently in the DOM, keyed by group key —
|
||||
* lets `_findLaneByKey()` resolve in O(1) instead of a container-wide
|
||||
* `querySelector` per lookup. Kept in sync with the DOM: entries are
|
||||
* added where lanes are created (`_appendItems`,
|
||||
* `_ensureJustAddedLane`) and the map is cleared on the full-container
|
||||
* wipes in `render()` / `clear()`; lanes are never removed
|
||||
* individually anywhere else.
|
||||
* @type {Map<string, HTMLElement>}
|
||||
*/
|
||||
this._lanes = new Map();
|
||||
|
||||
/**
|
||||
* Optional grouping-key resolver stored between `render()` / `append()`
|
||||
* calls so `addItem()` can place a new row in the correct swimlane
|
||||
@@ -203,6 +215,7 @@ export class ResourceListComponent {
|
||||
// Reset group tracking for the fresh render
|
||||
this._lastGroupKey = undefined;
|
||||
this._lastGroupEl = null;
|
||||
this._lanes.clear();
|
||||
this._groupFn = groupFn;
|
||||
this._groupLabelFn = groupLabelFn;
|
||||
this._headerNodeFn = headerNodeFn;
|
||||
@@ -244,6 +257,7 @@ export class ResourceListComponent {
|
||||
this._lastClickedIndex = -1;
|
||||
this._lastGroupKey = undefined;
|
||||
this._lastGroupEl = null;
|
||||
this._lanes.clear();
|
||||
this._groupFn = undefined;
|
||||
this._groupLabelFn = undefined;
|
||||
this._headerNodeFn = undefined;
|
||||
@@ -348,7 +362,8 @@ export class ResourceListComponent {
|
||||
// "New" swimlane, creating it on first call.
|
||||
const lane = this._ensureJustAddedLane();
|
||||
this._items.set(item.id, item);
|
||||
row = isFile ? this._createFileItem(/** @type {FileItem} */ (item)) : this._createFolderItem(/** @type {FolderItem} */ (item));
|
||||
const labels = this._buildItemLabels();
|
||||
row = isFile ? this._createFileItem(/** @type {FileItem} */ (item), labels) : this._createFolderItem(/** @type {FolderItem} */ (item), labels);
|
||||
lane.appendChild(row);
|
||||
} else {
|
||||
// Flat list (no grouping) — append at the end like before.
|
||||
@@ -394,6 +409,7 @@ export class ResourceListComponent {
|
||||
const lane = document.createElement('div');
|
||||
lane.className = 'resource-list__swimlane-group resource-list__swimlane-group--just-added';
|
||||
lane.dataset.groupKey = JUST_ADDED_KEY;
|
||||
this._lanes.set(JUST_ADDED_KEY, lane);
|
||||
|
||||
const header = document.createElement('div');
|
||||
header.className = 'resource-list__swimlane-header';
|
||||
@@ -420,13 +436,14 @@ export class ResourceListComponent {
|
||||
* Locate an on-screen swimlane wrapper by its group key. Returns
|
||||
* `null` when no swimlane currently matches.
|
||||
*
|
||||
* O(1) via the `_lanes` registry — see its declaration for how it is
|
||||
* kept in sync with the DOM.
|
||||
*
|
||||
* @param {string} key
|
||||
* @returns {HTMLElement | null}
|
||||
*/
|
||||
_findLaneByKey(key) {
|
||||
// CSS.escape covers arbitrary key shapes (dates with colons,
|
||||
// UUIDs with dashes, etc.) so the attribute selector is safe.
|
||||
return /** @type {HTMLElement | null} */ (this._container.querySelector(`.resource-list__swimlane-group[data-group-key="${CSS.escape(String(key))}"]`));
|
||||
return this._lanes.get(String(key)) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -523,6 +540,9 @@ export class ResourceListComponent {
|
||||
_appendItems(items, groupFn, groupLabelFn, headerNodeFn) {
|
||||
const fragment = document.createDocumentFragment();
|
||||
|
||||
// Resolve batch-invariant labels once, not once per row.
|
||||
const labels = this._buildItemLabels();
|
||||
|
||||
// Start from the persisted key so load-more pages continue seamlessly.
|
||||
let lastGroupKey = this._lastGroupKey;
|
||||
|
||||
@@ -545,11 +565,12 @@ export class ResourceListComponent {
|
||||
if (key !== null) {
|
||||
fragmentGroup = document.createElement('div');
|
||||
fragmentGroup.className = 'resource-list__swimlane-group';
|
||||
// Stamp the group key on the wrapper so `addItem()`
|
||||
// can locate this swimlane later via
|
||||
// `_findLaneByKey()` and append into it without a
|
||||
// full re-render.
|
||||
// Stamp the group key on the wrapper (handy in
|
||||
// devtools) and register it in `_lanes` so
|
||||
// `_findLaneByKey()` can locate this swimlane later
|
||||
// without a container-wide query.
|
||||
fragmentGroup.dataset.groupKey = key;
|
||||
this._lanes.set(key, fragmentGroup);
|
||||
fragmentGroup.appendChild(this._createGroupHeader(key, groupLabelFn, headerNodeFn));
|
||||
fragment.appendChild(fragmentGroup);
|
||||
}
|
||||
@@ -558,7 +579,9 @@ export class ResourceListComponent {
|
||||
|
||||
// Dispatch to the correct renderer: files have mime_type, folders do not.
|
||||
const isFile = 'mime_type' in item;
|
||||
const itemEl = isFile ? this._createFileItem(/** @type {FileItem} */ (item)) : this._createFolderItem(/** @type {FolderItem} */ (item));
|
||||
const itemEl = isFile
|
||||
? this._createFileItem(/** @type {FileItem} */ (item), labels)
|
||||
: this._createFolderItem(/** @type {FolderItem} */ (item), labels);
|
||||
|
||||
// Priority: live DOM group (load-more continuation) > current fragment group > bare container
|
||||
const target = liveGroup ?? fragmentGroup;
|
||||
@@ -600,12 +623,50 @@ export class ResourceListComponent {
|
||||
return el;
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} ItemLabels
|
||||
* @property {string} folderTypeLabel - Type-cell label for folders.
|
||||
* @property {string} customActionsHtml - Pre-rendered inline-action buttons.
|
||||
* @property {(category: string) => string} fileTypeLabel - Type-cell label
|
||||
* for a file category (memoized per batch).
|
||||
*/
|
||||
|
||||
/**
|
||||
* Resolve every per-row value that does not depend on the item once per
|
||||
* batch: the i18n lookups for the type cell and the custom-actions HTML
|
||||
* are identical for all 50 rows of a page, so repeating them in
|
||||
* `_createFileItem` / `_createFolderItem` was pure overhead. Built fresh
|
||||
* on every call (never cached on the instance), so a locale switch is
|
||||
* picked up naturally by the next render/append.
|
||||
*
|
||||
* @returns {ItemLabels}
|
||||
*/
|
||||
_buildItemLabels() {
|
||||
const fallbackTypeLabel = i18n.t('files.file_types.document');
|
||||
/** @type {Map<string, string>} */
|
||||
const byCategory = new Map();
|
||||
return {
|
||||
folderTypeLabel: i18n.t('files.file_types.folder'),
|
||||
customActionsHtml: this._renderCustomActions(),
|
||||
fileTypeLabel(category) {
|
||||
if (!category) return fallbackTypeLabel;
|
||||
let label = byCategory.get(category);
|
||||
if (label === undefined) {
|
||||
label = i18n.t(`files.file_types.${category.toLowerCase()}`) || category;
|
||||
byCategory.set(category, label);
|
||||
}
|
||||
return label;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a .file-item DOM element for a folder.
|
||||
* @param {FolderItem} folder
|
||||
* @param {ItemLabels} labels - Batch-invariant labels from `_buildItemLabels()`.
|
||||
* @returns {HTMLElement}
|
||||
*/
|
||||
_createFolderItem(folder) {
|
||||
_createFolderItem(folder, labels) {
|
||||
const cfg = this._cfg;
|
||||
const el = document.createElement('div');
|
||||
const modClass = cfg.itemModifierClass ? ` ${cfg.itemModifierClass}` : '';
|
||||
@@ -632,11 +693,11 @@ export class ResourceListComponent {
|
||||
</div>
|
||||
<div class="owner-cell${this._ownerVisible ? '' : ' hidden'}" data-owner-id="${escapeHtml(folder.owner_id || '')}"></div>
|
||||
${cfg.showPath ? `<div class="path-cell" title="${escapeHtml(folder.path || '')}">${escapeHtml(folder.path || '')}</div>` : ''}
|
||||
${cfg.showType ? `<div class="type-cell">${i18n.t('files.file_types.folder')}</div>` : ''}
|
||||
${cfg.showType ? `<div class="type-cell">${labels.folderTypeLabel}</div>` : ''}
|
||||
<div class="size-cell">--</div>
|
||||
<div class="date-cell">${formattedDate}</div>
|
||||
<div class="action-cell">
|
||||
${this._renderCustomActions()}
|
||||
${labels.customActionsHtml}
|
||||
${cfg.showFavorite ? `<button class="favorite-star${isFav ? ' active' : ''}"><i class="${isFav ? 'fas' : 'far'} fa-star"></i></button>` : ''}
|
||||
${cfg.showContextMenu ? '<button class="file-actions"><i class="fas fa-ellipsis-v"></i></button>' : ''}
|
||||
</div>
|
||||
@@ -649,12 +710,12 @@ export class ResourceListComponent {
|
||||
/**
|
||||
* Build a .file-item DOM element for a file.
|
||||
* @param {FileItem} file
|
||||
* @param {ItemLabels} labels - Batch-invariant labels from `_buildItemLabels()`.
|
||||
* @returns {HTMLElement}
|
||||
*/
|
||||
_createFileItem(file) {
|
||||
_createFileItem(file, labels) {
|
||||
const cfg = this._cfg;
|
||||
const cat = file.category || '';
|
||||
const typeLabel = cat ? i18n.t(`files.file_types.${cat.toLowerCase()}`) || cat : i18n.t('files.file_types.document');
|
||||
const typeLabel = labels.fileTypeLabel(file.category || '');
|
||||
const fileSize = file.size_formatted || formatFileSize(file.size);
|
||||
const dateVal = /** @type {Record<string,string>} */ (/** @type {unknown} */ (file))[cfg.dateField] ?? file.modified_at;
|
||||
const formattedDate = cfg.dateFormatter ? cfg.dateFormatter(dateVal) : formatDateTime(new Date(dateVal));
|
||||
@@ -685,7 +746,7 @@ export class ResourceListComponent {
|
||||
<div class="size-cell">${fileSize}</div>
|
||||
<div class="date-cell">${formattedDate}</div>
|
||||
<div class="action-cell">
|
||||
${this._renderCustomActions()}
|
||||
${labels.customActionsHtml}
|
||||
${cfg.showFavorite ? `<button class="favorite-star${isFav ? ' active' : ''}"><i class="${isFav ? 'fas' : 'far'} fa-star"></i></button>` : ''}
|
||||
${cfg.showContextMenu ? '<button class="file-actions"><i class="fas fa-ellipsis-v"></i></button>' : ''}
|
||||
</div>
|
||||
|
||||
@@ -2,29 +2,47 @@ import { getCsrfHeaders } from '../core/csrf.js';
|
||||
|
||||
/** @import {FileItem} from '../core/types.js' */
|
||||
|
||||
// IMPORTANT: absolute paths so the dynamic import resolves correctly both in
|
||||
// dev mode (native ESM, module at /js/features/thumbnail.js) and in release
|
||||
// mode (IIFE bundle at /js/app.{hash}.js — relative '../vendors/…' would
|
||||
// incorrectly resolve to /vendors/… instead of /js/vendors/…).
|
||||
const PDFJS_LIB_URL = '/js/vendors/pdf.min.mjs';
|
||||
const PDFJS_WORKER_URL = '/js/vendors/pdf.worker.min.mjs';
|
||||
|
||||
/**
|
||||
* use any type so tsc will not scan library
|
||||
* @type {any}
|
||||
* Memoized import of pdf.min.mjs (in-flight or settled).
|
||||
* use any type so tsc will not scan library.
|
||||
* Reset to null on failure so a later call retries (e.g. transient offline).
|
||||
* @type {Promise<any> | null}
|
||||
*/
|
||||
let _pdfjsLib = null;
|
||||
let _pdfjsLibPromise = null;
|
||||
|
||||
/** True once the worker script warm-up fetch has completed successfully. */
|
||||
let _pdfWorkerWarmed = false;
|
||||
|
||||
// TODO: do we need to add a max concurrncy ?
|
||||
|
||||
/**
|
||||
* Lazy-loads pdf.min.mjs on first use via dynamic import so it is never
|
||||
* bundled into the IIFE (it uses top-level await which breaks IIFE wrapping).
|
||||
* Memoizing the promise (rather than the resolved module) lets concurrent
|
||||
* callers — e.g. `preloadPdf()` racing the first real thumbnail — share a
|
||||
* single network fetch.
|
||||
* @returns {Promise<any>}
|
||||
*/
|
||||
async function getPdfjsLib() {
|
||||
if (_pdfjsLib) return _pdfjsLib;
|
||||
// IMPORTANT: use an absolute path so the import resolves correctly both in
|
||||
// dev mode (native ESM, module at /js/features/thumbnail.js) and in release
|
||||
// mode (IIFE bundle at /js/app.{hash}.js — relative '../vendors/…' would
|
||||
// incorrectly resolve to /vendors/… instead of /js/vendors/…).
|
||||
const lib = '/js/vendors/pdf.min.mjs';
|
||||
_pdfjsLib = /** @type {any} */ (await import(lib));
|
||||
_pdfjsLib.GlobalWorkerOptions.workerSrc = '/js/vendors/pdf.worker.min.mjs';
|
||||
return _pdfjsLib;
|
||||
function getPdfjsLib() {
|
||||
if (!_pdfjsLibPromise) {
|
||||
_pdfjsLibPromise = import(PDFJS_LIB_URL)
|
||||
.then((lib) => {
|
||||
lib.GlobalWorkerOptions.workerSrc = PDFJS_WORKER_URL;
|
||||
return lib;
|
||||
})
|
||||
.catch((err) => {
|
||||
_pdfjsLibPromise = null; // allow retry after a failed load
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
return _pdfjsLibPromise;
|
||||
}
|
||||
|
||||
export const thumbnail = {
|
||||
@@ -43,6 +61,33 @@ export const thumbnail = {
|
||||
return false;
|
||||
},
|
||||
|
||||
/**
|
||||
* Fire-and-forget warm-up of the pdf.js stack (module + worker script).
|
||||
*
|
||||
* Called the moment a PDF row enters the DOM (see resourceIcon.js), so
|
||||
* the ~1.3 MB library downloads in the background while the user is
|
||||
* still looking at the list — instead of stalling the first thumbnail
|
||||
* render on it. Idempotent and cheap after the first call, and only
|
||||
* folders that actually contain PDFs ever pay the download.
|
||||
*/
|
||||
preloadPdf() {
|
||||
// Module (≈300 KB): shares the memoized promise with real users.
|
||||
getPdfjsLib().catch(() => {
|
||||
/* transient failure — the first real use retries */
|
||||
});
|
||||
|
||||
// Worker (≈1 MB): pdf.js only fetches it via `new Worker(...)` on the
|
||||
// first getDocument(), so prime the HTTP cache with a plain fetch.
|
||||
// Reading the body ensures the download completes and is cacheable.
|
||||
if (_pdfWorkerWarmed) return;
|
||||
_pdfWorkerWarmed = true;
|
||||
fetch(PDFJS_WORKER_URL)
|
||||
.then((r) => (r.ok ? r.blob() : Promise.reject(new Error(`HTTP ${r.status}`))))
|
||||
.catch(() => {
|
||||
_pdfWorkerWarmed = false; // allow retry on a later sighting
|
||||
});
|
||||
},
|
||||
|
||||
// TODO: use these informations from server ?
|
||||
SIZES: {
|
||||
icon: { width: 150, height: 150 },
|
||||
|
||||
@@ -151,6 +151,10 @@ function switchTab(name, el) {
|
||||
}
|
||||
|
||||
activeTabName = name;
|
||||
// The migration auto-poll only makes sense while the Storage tab is
|
||||
// visible — without this it would keep hitting the API every 2 s
|
||||
// (and updating hidden DOM) for as long as a migration runs.
|
||||
if (name !== 'storage') stopMigrationPolling();
|
||||
if (name === 'users') loadUsers();
|
||||
if (name === 'dashboard') loadDashboard();
|
||||
if (name === 'storage') loadStorage();
|
||||
@@ -908,6 +912,22 @@ async function testStorageConnection() {
|
||||
/** @type {ReturnType<typeof setInterval> | null} */
|
||||
let migrationPollTimer = null;
|
||||
|
||||
/**
|
||||
* Stop the 2 s migration auto-poll if it is armed.
|
||||
*
|
||||
* Called when the poll observes a non-running status, when a poll request
|
||||
* fails (an expired admin session would otherwise be retried every 2 s
|
||||
* forever), and when the user leaves the Storage tab. Re-entering the tab
|
||||
* re-arms it via `loadStorage()` → `loadMigrationStatus()` while a
|
||||
* migration is running.
|
||||
*/
|
||||
function stopMigrationPolling() {
|
||||
if (migrationPollTimer) {
|
||||
clearInterval(migrationPollTimer);
|
||||
migrationPollTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} msg
|
||||
* @param {string} type
|
||||
@@ -979,7 +999,12 @@ async function loadMigrationStatus() {
|
||||
headers: headers(),
|
||||
credentials: 'same-origin'
|
||||
});
|
||||
if (!resp.ok) return;
|
||||
if (!resp.ok) {
|
||||
// Don't keep hammering a failing endpoint (e.g. expired session);
|
||||
// any migration button or tab re-entry re-arms the poll.
|
||||
stopMigrationPolling();
|
||||
return;
|
||||
}
|
||||
const m = await resp.json();
|
||||
updateMigrationUI(m);
|
||||
|
||||
@@ -988,9 +1013,8 @@ async function loadMigrationStatus() {
|
||||
if (!migrationPollTimer) {
|
||||
migrationPollTimer = setInterval(loadMigrationStatus, 2000);
|
||||
}
|
||||
} else if (migrationPollTimer) {
|
||||
clearInterval(migrationPollTimer);
|
||||
migrationPollTimer = null;
|
||||
} else {
|
||||
stopMigrationPolling();
|
||||
}
|
||||
} catch (_e) {
|
||||
/* ignore */
|
||||
|
||||
Reference in New Issue
Block a user