fix(thumbnails): two defects the copy test exposed
1. Per-file overrides must beat the content tier in RAM as well as on
disk. The content-keyed lookup ran first, so a thumbnail already
rendered from the file's content sat in RAM under content(blob_hash)
and shadowed a preview uploaded afterwards — permanently. Invisible
before the moka rekey, because both lived under one file-id key and
the upload simply overwrote the render. Order is now uniform:
per-file RAM, per-file disk (ext-), per-file DB, then content RAM,
blob-hash disk, derived blob.
2. store_attached_blob never wrote a row. Its RETURNING clause compared
the stored hash against EXCLUDED, and PostgreSQL only permits
EXCLUDED in the SET and WHERE of DO UPDATE — a runtime syntax error
on every call. The superseded hash now comes from a SELECT taken
before the upsert; losing that race leaves one stale reference, which
the manifest recompute reports, rather than anything being lost.
The second hid behind the first for a whole cycle, and behind
`ext-{file_id}.jpg`: the ORIGINAL kept serving its uploaded preview from
local disk, so the feature looked healthy. Only a copy, which has a
different file_id and therefore no ext- file, depends on the row — and
the row was never there. The handler's best-effort warn! completed the
disguise, so it is now error!: a failure there means copies silently
lose the preview, and nothing else signals it.
This commit is contained in:
@@ -607,10 +607,29 @@ impl DedupService {
|
|||||||
.await?;
|
.await?;
|
||||||
let attached_hash = stored.hash().to_string();
|
let attached_hash = stored.hash().to_string();
|
||||||
|
|
||||||
// `previous` is the hash this row pointed at before, when it existed
|
// Read the hash being superseded BEFORE upserting.
|
||||||
// and differed — the reference to release once the row no longer
|
//
|
||||||
// holds it.
|
// It cannot come from `RETURNING`: PostgreSQL only permits `EXCLUDED`
|
||||||
let previous: Option<(Option<String>,)> = sqlx::query_as(
|
// in the `SET` and `WHERE` of `DO UPDATE`, so a RETURNING clause
|
||||||
|
// comparing old against new is a syntax error — and one that surfaces
|
||||||
|
// only at runtime, where this method's best-effort caller swallows it
|
||||||
|
// into a warning while the sidecar keeps the feature looking healthy.
|
||||||
|
//
|
||||||
|
// The gap between this SELECT and the upsert is benign: losing the
|
||||||
|
// race leaves one stale reference, which the manifest recompute
|
||||||
|
// reports rather than anything being lost or served wrongly.
|
||||||
|
let previous: Option<(String,)> = sqlx::query_as(
|
||||||
|
"SELECT blob_hash FROM storage.file_attached_blobs
|
||||||
|
WHERE file_id = $1::uuid AND kind = $2 AND variant = $3",
|
||||||
|
)
|
||||||
|
.bind(file_id)
|
||||||
|
.bind(kind)
|
||||||
|
.bind(variant)
|
||||||
|
.fetch_optional(self.pool.as_ref())
|
||||||
|
.await
|
||||||
|
.map_err(|e| DomainError::internal_error("Dedup", format!("read attached blob: {e}")))?;
|
||||||
|
|
||||||
|
sqlx::query(
|
||||||
"INSERT INTO storage.file_attached_blobs
|
"INSERT INTO storage.file_attached_blobs
|
||||||
(file_id, kind, variant, blob_hash, content_type, uploaded_by)
|
(file_id, kind, variant, blob_hash, content_type, uploaded_by)
|
||||||
VALUES ($1::uuid, $2, $3, $4, $5, $6)
|
VALUES ($1::uuid, $2, $3, $4, $5, $6)
|
||||||
@@ -618,8 +637,7 @@ impl DedupService {
|
|||||||
SET blob_hash = EXCLUDED.blob_hash,
|
SET blob_hash = EXCLUDED.blob_hash,
|
||||||
content_type = EXCLUDED.content_type,
|
content_type = EXCLUDED.content_type,
|
||||||
uploaded_by = EXCLUDED.uploaded_by,
|
uploaded_by = EXCLUDED.uploaded_by,
|
||||||
created_at = now()
|
created_at = now()",
|
||||||
RETURNING NULLIF(storage.file_attached_blobs.blob_hash, EXCLUDED.blob_hash)",
|
|
||||||
)
|
)
|
||||||
.bind(file_id)
|
.bind(file_id)
|
||||||
.bind(kind)
|
.bind(kind)
|
||||||
@@ -627,14 +645,14 @@ impl DedupService {
|
|||||||
.bind(&attached_hash)
|
.bind(&attached_hash)
|
||||||
.bind(content_type)
|
.bind(content_type)
|
||||||
.bind(uploaded_by)
|
.bind(uploaded_by)
|
||||||
.fetch_optional(self.pool.as_ref())
|
.execute(self.pool.as_ref())
|
||||||
.await
|
.await
|
||||||
.map_err(|e| DomainError::internal_error("Dedup", format!("record attached blob: {e}")))?;
|
.map_err(|e| DomainError::internal_error("Dedup", format!("record attached blob: {e}")))?;
|
||||||
|
|
||||||
// A replaced row's old blob loses its only reference from here. Not
|
// A replaced row's old blob loses its only reference from here. Not
|
||||||
// releasing it would pin those bytes forever — nothing else points at
|
// releasing it would pin those bytes forever — nothing else points at
|
||||||
// a superseded preview.
|
// a superseded preview.
|
||||||
if let Some((Some(old_hash),)) = previous
|
if let Some((old_hash,)) = previous
|
||||||
&& old_hash != attached_hash
|
&& old_hash != attached_hash
|
||||||
&& let Err(e) = self.remove_reference(&old_hash).await
|
&& let Err(e) = self.remove_reference(&old_hash).await
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -550,19 +550,22 @@ impl ThumbnailService {
|
|||||||
// today's behaviour, which is what the port impl wants.
|
// today's behaviour, which is what the port impl wants.
|
||||||
dedup: Option<&DedupService>,
|
dedup: Option<&DedupService>,
|
||||||
) -> Option<Bytes> {
|
) -> Option<Bytes> {
|
||||||
// 1. Check in-memory cache.
|
// A file-specific override beats anything derived from the content,
|
||||||
|
// and that has to hold at EVERY tier — including RAM. Checking the
|
||||||
|
// content-keyed entry first would let a previously-rendered
|
||||||
|
// thumbnail shadow a preview the user has since uploaded: the render
|
||||||
|
// is cached under `content(hash)`, the upload lands under
|
||||||
|
// `external(file_id)`, and the content key would win forever.
|
||||||
//
|
//
|
||||||
// Keyed by content, so a caller that did not resolve the hash cannot
|
// So the order is: per-file RAM, per-file disk, per-file DB, then the
|
||||||
// consult this tier — it falls through to disk, which is correct
|
// content-keyed tiers. Same precedence as the disk tiers below, just
|
||||||
// rather than merely acceptable: a file-id key would be the stale
|
// applied one level up.
|
||||||
// entry the content key exists to avoid. Every HTTP path passes the
|
|
||||||
// hash (both handlers resolve it to build the ETag), so the fall
|
// 1. Per-file override in RAM (uploaded preview / video frame).
|
||||||
// through is confined to internal callers that never had one.
|
if let Some(bytes) = self
|
||||||
if let Some(hash) = blob_hash
|
.cache
|
||||||
&& let Some(bytes) = self
|
.get(&ThumbnailCacheKey::external(file_id, size))
|
||||||
.cache
|
.await
|
||||||
.get(&ThumbnailCacheKey::content(hash, size, format))
|
|
||||||
.await
|
|
||||||
&& !bytes.is_empty()
|
&& !bytes.is_empty()
|
||||||
{
|
{
|
||||||
return Some(bytes);
|
return Some(bytes);
|
||||||
@@ -612,8 +615,26 @@ impl ThumbnailService {
|
|||||||
return Some(bytes);
|
return Some(bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Check disk for blob-hash thumbnails (needs blob_hash to locate)
|
// 3. Content-keyed RAM tier. Below the per-file tiers by the rule
|
||||||
|
// above; still ahead of every disk read.
|
||||||
|
//
|
||||||
|
// A caller that did not resolve the hash cannot consult it and
|
||||||
|
// falls through to disk. That is correct rather than merely
|
||||||
|
// acceptable: a file-id key here would be the stale entry content
|
||||||
|
// keying exists to avoid. Both HTTP handlers resolve the hash to
|
||||||
|
// build the ETag, so the fall-through is confined to internal
|
||||||
|
// callers that never had one.
|
||||||
let hash = blob_hash?;
|
let hash = blob_hash?;
|
||||||
|
if let Some(bytes) = self
|
||||||
|
.cache
|
||||||
|
.get(&ThumbnailCacheKey::content(hash, size, format))
|
||||||
|
.await
|
||||||
|
&& !bytes.is_empty()
|
||||||
|
{
|
||||||
|
return Some(bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Check disk for blob-hash thumbnails (needs blob_hash to locate)
|
||||||
let thumb_path = self.get_thumbnail_path(hash, size, format);
|
let thumb_path = self.get_thumbnail_path(hash, size, format);
|
||||||
if let Ok(data) = fs::read(&thumb_path).await {
|
if let Ok(data) = fs::read(&thumb_path).await {
|
||||||
let bytes = Bytes::from(data);
|
let bytes = Bytes::from(data);
|
||||||
|
|||||||
@@ -768,11 +768,15 @@ impl FileHandler {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::warn!(
|
// ERROR, not WARN: the sidecar keeps the feature looking healthy
|
||||||
|
// on this box, so nothing else signals that copies are silently
|
||||||
|
// losing the preview. A syntax error in the upsert hid behind a
|
||||||
|
// warning for an entire test cycle exactly this way.
|
||||||
|
tracing::error!(
|
||||||
target: "oxicloud::dedup",
|
target: "oxicloud::dedup",
|
||||||
error = %e,
|
error = %e,
|
||||||
file_id = %id,
|
file_id = %id,
|
||||||
"failed to record attached thumbnail; sidecar written, copies will not inherit it"
|
"failed to record attached thumbnail; sidecar written, copies will NOT inherit it"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user