feat(recoverable-job): add backend_consistency (storage)

This commit is contained in:
Edouard Vanbelle
2026-07-29 23:40:19 +02:00
parent 07beb461d6
commit 507bc2e98d
11 changed files with 844 additions and 0 deletions
@@ -421,4 +421,97 @@ impl BlobStorageBackend for S3BlobBackend {
fn local_blob_path(&self, _hash: &str) -> Option<PathBuf> {
None // Remote backend — no local path
}
/// Enumerate blobs via S3 `ListObjectsV2`. Cursor is the S3
/// continuation token verbatim (opaque). Filter: keys must
/// match `<xx>/<64-hex>.blob` — matches how `blob_key` writes
/// them — so any future non-blob namespace living in the same
/// bucket (e.g. `thumbnails/<hash>.jpg`) is skipped
/// automatically. No prefix passed to S3 so we get everything
/// in one paginated scan; the client-side filter enforces
/// correctness.
fn list_blob_hashes(
&self,
cursor: Option<String>,
limit: usize,
) -> Pin<
Box<
dyn std::future::Future<
Output = Result<
crate::application::ports::blob_storage_ports::BlobListPage,
DomainError,
>,
> + Send
+ '_,
>,
> {
use crate::application::ports::blob_storage_ports::{
BackendBlobEntry, BackendUnknownEntry, BlobListPage,
};
Box::pin(async move {
let mut req = self
.client
.list_objects_v2()
.bucket(&self.bucket)
.max_keys(limit.min(1000) as i32);
if let Some(c) = cursor {
req = req.continuation_token(c);
}
let resp = req.send().await.map_err(|e| {
DomainError::new(
ErrorKind::InternalError,
"Blob",
format!("S3 ListObjectsV2 failed: {e}"),
)
})?;
let objects = resp.contents.unwrap_or_default();
let mut blobs: Vec<BackendBlobEntry> = Vec::with_capacity(objects.len());
let mut unknowns: Vec<BackendUnknownEntry> = Vec::new();
for obj in objects {
let Some(key) = obj.key else { continue };
let mtime = obj.last_modified.and_then(|ts| {
let secs = ts.secs();
let nsecs = ts.subsec_nanos();
chrono::DateTime::<chrono::Utc>::from_timestamp(secs, nsecs)
});
// Canonical S3 key shape: `<xx>/<64-hex>.blob`.
// Anything else is a sidecar or foreign namespace
// (e.g. future `thumbnails/<hash>.jpg` if Ed adds
// that) — surface as an unknown so operators know
// it's there. Recovery framework can decide per-
// pattern how to act.
let is_canonical = key.split_once('/').and_then(|(prefix, rest)| {
if prefix.len() != 2 || !prefix.chars().all(|c| c.is_ascii_hexdigit()) {
return None;
}
rest.strip_suffix(".blob")
.filter(|stem| {
stem.len() == 64 && stem.chars().all(|c| c.is_ascii_hexdigit())
})
.map(|s| s.to_string())
});
match is_canonical {
Some(hash) => blobs.push(BackendBlobEntry { hash, mtime }),
None => unknowns.push(BackendUnknownEntry { path: key, mtime }),
}
}
let next_cursor = if resp.is_truncated.unwrap_or(false) {
resp.next_continuation_token
} else {
None
};
Ok(BlobListPage {
blobs,
unknowns,
next_cursor,
})
})
}
}