Instant upload: register already-owned content by hash, zero bytes on the wire
Phase 0 of the delta-sync plan. Re-uploading a file the user already has (another device, a restore, a duplicate) used to transfer every byte just for the server to discard them as a dedup hit. The frontend now computes the file's BLAKE3 locally and, on a hit, registers the file with a single ~150-byte metadata call. Server — POST /api/files/by-hash: - All checks live in the application service per the AuthZ rule: Create permission on the target folder via the authorization engine, hash ownership via the existing user-scoped query (a non-owned hash returns 404 — same shape as "no such blob" — and emits an instant_upload.rejected audit event), quota on the logical size. - On success: one ref_count bump + the existing save_file_with_blob row registration (compensation included); is_new_blob=false so lifecycle hooks skip thumbnail regeneration. ~10 ms warm. - The storage-usage service is now built before the application services and injected, instead of only living on AppState. Client — WASM BLAKE3 + worker: - wasm/oxicloud-hash: the exact same blake3 crate the server uses, compiled with WASM SIMD128 (~660 MB/s measured) so browser hashes match server content addresses bit for bit. Built by scripts/build-wasm.sh; the artifacts (45 KB wasm + 8 KB glue) are vendored like pdf.js — no npm dependencies, no wasm toolchain needed for regular builds. - static/js/workers/hashWorker.js streams the File in 8 MiB slices off the main thread (constant RAM at any file size). - features/files/instantUpload.js orchestrates: threshold (8 MiB — below it the round-trips cost more than the bytes), user-scoped /api/dedup/check, by-hash registration, and silent fallback to the normal byte upload on any miss, race or unsupported environment. Wired into both uploadFiles and uploadFolderEntries. - biome.json vendors exclusion fixed to cover nested directories (previous vendors were .mjs and never matched the *.js include). Verified end-to-end against PostgreSQL 16: node-driven WASM hash equals the server's content_hash for a 20 MB file; by-hash returns 201 in ~10 ms warm with a 151-byte request (vs 20,971,873 bytes for the byte upload); the copy downloads byte-identical and the manifest ref_count goes 1→2; a second user probing the same hash gets exists:false and 404 plus the audit line; duplicate name → 409, malformed hash → 400; worker and wasm are served with correct MIME (application/wasm). https://claude.ai/code/session_01WdNenpnujNR2sc32XVvwfS
This commit is contained in:
Claude
@@ -1,14 +1,19 @@
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::application::dtos::file_dto::FileDto;
|
||||
use crate::application::ports::authorization_ports::AuthorizationEngine;
|
||||
use crate::application::ports::file_lifecycle::FileLifecycleHook;
|
||||
use crate::application::ports::file_ports::{FileUploadUseCase, StoredBlob};
|
||||
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort};
|
||||
use crate::application::ports::storage_ports::{FileReadPort, FileWritePort, StorageUsagePort};
|
||||
use crate::application::services::storage_usage_service::StorageUsageService;
|
||||
use crate::common::errors::DomainError;
|
||||
use crate::domain::services::authorization::{Permission, Resource, Subject};
|
||||
use crate::infrastructure::repositories::pg::FileBlobReadRepository;
|
||||
use crate::infrastructure::repositories::pg::FileBlobWriteRepository;
|
||||
use crate::infrastructure::services::dedup_service::DedupService;
|
||||
use crate::infrastructure::services::file_content_cache::FileContentCache;
|
||||
use crate::infrastructure::services::pg_acl_engine::PgAclEngine;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// Helper function to extract username from folder path string.
|
||||
@@ -49,6 +54,18 @@ pub struct FileUploadService {
|
||||
content_cache: Option<Arc<FileContentCache>>,
|
||||
/// Single lifecycle dispatcher — fires on_file_created / on_file_updated.
|
||||
file_lifecycle_hook: Option<Arc<dyn FileLifecycleHook>>,
|
||||
/// Dependencies of the instant-upload path
|
||||
/// (`create_file_from_owned_blob_with_perms`); `None` in minimal test
|
||||
/// wiring.
|
||||
instant_upload: Option<InstantUploadDeps>,
|
||||
}
|
||||
|
||||
/// Everything the instant-upload path needs beyond the upload service's own
|
||||
/// ports: permission checks, the dedup index, and quota enforcement.
|
||||
struct InstantUploadDeps {
|
||||
authz: Arc<PgAclEngine>,
|
||||
dedup: Arc<DedupService>,
|
||||
quota: Arc<StorageUsageService>,
|
||||
}
|
||||
|
||||
impl FileUploadService {
|
||||
@@ -60,6 +77,7 @@ impl FileUploadService {
|
||||
storage_usage_service: None,
|
||||
content_cache: None,
|
||||
file_lifecycle_hook: None,
|
||||
instant_upload: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,9 +92,26 @@ impl FileUploadService {
|
||||
storage_usage_service: None,
|
||||
content_cache: None,
|
||||
file_lifecycle_hook: None,
|
||||
instant_upload: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Wires the authorization engine, dedup index and quota service that
|
||||
/// power the instant-upload path.
|
||||
pub fn with_instant_upload(
|
||||
mut self,
|
||||
authz: Arc<PgAclEngine>,
|
||||
dedup: Arc<DedupService>,
|
||||
quota: Arc<StorageUsageService>,
|
||||
) -> Self {
|
||||
self.instant_upload = Some(InstantUploadDeps {
|
||||
authz,
|
||||
dedup,
|
||||
quota,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Configures the content cache for invalidation on file updates.
|
||||
pub fn with_content_cache(mut self, cache: Arc<FileContentCache>) -> Self {
|
||||
self.content_cache = Some(cache);
|
||||
@@ -98,6 +133,111 @@ impl FileUploadService {
|
||||
self
|
||||
}
|
||||
|
||||
// ── Instant upload (zero content bytes) ──────────────────────
|
||||
|
||||
/// Register a new file row pointing at a blob the caller **already
|
||||
/// owns** — the instant-upload path: the client proved it has the
|
||||
/// content by hash, so no bytes travel and no chunk is written. Pure
|
||||
/// metadata: one ref_count bump + one row INSERT.
|
||||
///
|
||||
/// Security model (mirrors `GET /api/dedup/check/{hash}`):
|
||||
/// - The caller must have `Create` permission on the target folder.
|
||||
/// - The hash is only claimable when the caller owns at least one
|
||||
/// non-trashed file referencing it — never a global content oracle.
|
||||
/// A non-owned hash returns `NotFound` (anti-enumeration: same shape
|
||||
/// as "no such blob") and emits an `instant_upload.rejected` audit
|
||||
/// event with the real reason.
|
||||
/// - Quota is enforced on the logical size, exactly like a byte upload.
|
||||
pub async fn create_file_from_owned_blob_with_perms(
|
||||
&self,
|
||||
caller_id: Uuid,
|
||||
name: String,
|
||||
folder_id: String,
|
||||
hash: &str,
|
||||
) -> Result<FileDto, DomainError> {
|
||||
let Some(InstantUploadDeps {
|
||||
authz,
|
||||
dedup,
|
||||
quota,
|
||||
}) = &self.instant_upload
|
||||
else {
|
||||
return Err(DomainError::internal_error(
|
||||
"FileUpload",
|
||||
"instant upload is not wired (authz/dedup/quota missing)",
|
||||
));
|
||||
};
|
||||
|
||||
// ── AuthZ: Create on the target folder ───────────────────
|
||||
let folder_uuid = Uuid::parse_str(&folder_id)
|
||||
.map_err(|_| DomainError::not_found("Folder", folder_id.clone()))?;
|
||||
authz
|
||||
.require(
|
||||
Subject::User(caller_id),
|
||||
Permission::Create,
|
||||
Resource::Folder(folder_uuid),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// ── Ownership: only blobs the caller can already read ────
|
||||
if !dedup
|
||||
.user_owns_blob_reference(hash, &caller_id.to_string())
|
||||
.await
|
||||
{
|
||||
tracing::info!(
|
||||
target: "audit",
|
||||
event = "instant_upload.rejected",
|
||||
reason = "hash_not_owned",
|
||||
caller_id = %caller_id,
|
||||
blob_hash = %hash,
|
||||
"👮🏻♂️ Instant upload rejected: caller owns no file referencing the claimed hash",
|
||||
);
|
||||
return Err(DomainError::not_found("Blob", hash));
|
||||
}
|
||||
|
||||
let Some(metadata) = dedup.get_blob_metadata(hash).await else {
|
||||
// Lost a race with the last-reference delete — same shape as
|
||||
// "never existed".
|
||||
return Err(DomainError::not_found("Blob", hash));
|
||||
};
|
||||
|
||||
// ── Quota on the logical size, before taking any reference ──
|
||||
quota.check_storage_quota(caller_id, metadata.size).await?;
|
||||
|
||||
// The manifest knows the original content type; fall back to the
|
||||
// new name's extension when the stored one is generic.
|
||||
let claimed = metadata.content_type.as_deref().unwrap_or("");
|
||||
let content_type =
|
||||
match crate::common::mime_detect::refine_content_type(&[], &name, claimed) {
|
||||
ct if ct.is_empty() => "application/octet-stream".to_string(),
|
||||
ct => ct,
|
||||
};
|
||||
|
||||
// Take the reference the row registration will consume (it releases
|
||||
// it again on any failure). A concurrent GC between the ownership
|
||||
// check and this bump surfaces as NotFound — the client falls back
|
||||
// to a normal byte upload.
|
||||
dedup.add_reference(hash).await?;
|
||||
|
||||
let dto = self
|
||||
.upload_file_streaming(
|
||||
name,
|
||||
Some(folder_id),
|
||||
content_type,
|
||||
StoredBlob {
|
||||
hash: hash.to_string(),
|
||||
size: metadata.size,
|
||||
is_new_blob: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!(
|
||||
"⚡ INSTANT UPLOAD: {} ({} bytes, 0 transferred, ID: {})",
|
||||
dto.name, metadata.size, dto.id
|
||||
);
|
||||
Ok(dto)
|
||||
}
|
||||
|
||||
// ── private helpers ──────────────────────────────────────────
|
||||
|
||||
/// Optionally update storage usage after a successful upload.
|
||||
|
||||
Reference in New Issue
Block a user