fix(name duplicate): fixed via NFC normalisation
TL;DR:
fix duplicate filename via:
```
docker exec <container> migrate-nfc-filenames --dry-run # preview
docker exec <container> migrate-nfc-filenames # execute
```
== issue ==
Last week I uploaded Capture d'écran 2026-06-03 à 20.04.24.png from the web. It synced down to Nextcloud on my Mac. Two minutes later, the Web UI was showing the file twice.
Both rows had:
- the same name
- the same size
- the same content hash
So why two rows? Because to PostgreSQL, the names weren't the same.
Web upload (browser → Postgres):
"é" stored as 1 codepoint (U+00E9) bytes: c3 a9 ← NFC
NiextCloud client (macOS → Postgres):
"é" stored as 2 codepoints (e + U+0301) bytes: 65 cc 81 ← NFD
macOS's APFS keeps filenames in NFD (decomposed); browsers send NFC (composed). Visually é and é are identical. To WHERE name = $1 they're two different keys. Our UNIQUE index on (folder_id, name, user_id) never fired — and the row count quietly drifted every time a Mac user touched an accented
filename.
== The fix is two halves ==
1. No new duplicates — every name-receiving boundary (file upload, NC PUT, rename, MOVE, path lookup) now NFC-normalizes before touching the database. The storage invariant becomes "every stored name is NFC".
2. Clean up existing data — one-shot migrate-nfc-filenames binary walks storage.files, NFC-normalizes any non-NFC row, and resolves the collisions we've accumulated. Same-content duplicates go to trash (recoverable); different-content collisions get renamed with a .duplicate suffix.
== use of the clean up ==
example of use (do not forget to define env **DATABASE_URL**)
either
`cargo run --bin migrate-nfc-filenames -- --dry-run`
or
`cargo build --bin migrate-nfc-filenames`
`./target/debug/migrate-nfc-filenames --dry-run`
example:
```
% ./target/debug/migrate-nfc-filenames --dry-run
=== NFC filename migration (DRY RUN — no writes) ===
Loaded 543 non-trashed file rows
NORMALIZE 163451b5-5e6c-404b-9b1e-f4b01a2b7269 user=42433185-4717-416d-9a15-4580fff171ec 'Capture d’écran 2026-03-20 à 14.44.50.png' → 'Capture d’écran 2026-03-20 à 14.44.50.png'
NORMALIZE 827dddec-4dd5-48c2-a120-dec5289f7d29 user=969deca6-7935-4f12-a430-4d636b62fa3e 'Capture d’écran 2026-04-03 à 15.43.38.png' → 'Capture d’écran 2026-04-03 à 15.43.38.png'
NORMALIZE 09559934-a620-472d-9ba8-fc3cfeb6dc6f user=a0643a21-0092-4a84-9dde-7ac4e76bc1a5 'Capture d’écran 2026-06-03 à 20.05.38.png' → 'Capture d’écran 2026-06-03 à 20.05.38.png'
NORMALIZE 5ce6dbf9-0562-4758-8783-671aa9069590 user=a0643a21-0092-4a84-9dde-7ac4e76bc1a5 'Capture d’écran 2026-06-05 à 11.07.25.png' → 'Capture d’écran 2026-06-05 à 11.07.25.png'
DEDUP newer=26bcf82b-99cc-45c8-9d69-dd7e5c4484ff (trash, same blob) older=df3adc67-a778-424d-a817-b930c75f3b06 user=a0643a21-0092-4a84-9dde-7ac4e76bc1a5 hash=0d2cc7b0ffce2850
=== Summary ===
scanned : 543
already in NFC : 538
normalized in place (no collision) : 4
dedup-trashed (same content) : 1
renamed to .duplicate : 0
DRY RUN — no rows were written. Re-run without --dry-run to apply.
```
once valid remove --dry-run
This commit is contained in:
Edouard Vanbelle
@@ -1,6 +1,8 @@
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::services::path_service::{StoragePath, validate_storage_name};
|
||||
use crate::domain::services::path_service::{
|
||||
StoragePath, normalize_storage_name, validate_storage_name,
|
||||
};
|
||||
|
||||
// Re-export entity errors from the centralized module
|
||||
pub use super::entity_errors::{FileError, FileResult};
|
||||
@@ -107,6 +109,7 @@ impl File {
|
||||
mime_type: String,
|
||||
folder_id: Option<String>,
|
||||
) -> FileResult<Self> {
|
||||
let name = normalize_storage_name(&name);
|
||||
if let Err(reason) = validate_storage_name(&name) {
|
||||
return Err(FileError::InvalidFileName(format!("{name}: {reason}")));
|
||||
}
|
||||
@@ -143,6 +146,7 @@ impl File {
|
||||
created_at: u64,
|
||||
modified_at: u64,
|
||||
) -> FileResult<Self> {
|
||||
let name = normalize_storage_name(&name);
|
||||
if let Err(reason) = validate_storage_name(&name) {
|
||||
return Err(FileError::InvalidFileName(format!("{name}: {reason}")));
|
||||
}
|
||||
@@ -204,6 +208,7 @@ impl File {
|
||||
owner_id: Option<Uuid>,
|
||||
blob_hash: String,
|
||||
) -> FileResult<Self> {
|
||||
let name = normalize_storage_name(&name);
|
||||
if let Err(reason) = validate_storage_name(&name) {
|
||||
return Err(FileError::InvalidFileName(format!("{name}: {reason}")));
|
||||
}
|
||||
@@ -345,7 +350,11 @@ impl File {
|
||||
// Create storage_path from string
|
||||
let storage_path = StoragePath::from_string(&path);
|
||||
|
||||
// Create directly without validation to avoid errors in DTO conversions
|
||||
// Create directly without validation to avoid errors in DTO
|
||||
// conversions. Still NFC-normalize so even DTO-reconstructed
|
||||
// entities maintain the storage invariant.
|
||||
let name = normalize_storage_name(&name);
|
||||
|
||||
Self {
|
||||
id,
|
||||
name,
|
||||
@@ -365,6 +374,7 @@ impl File {
|
||||
|
||||
/// Creates a new version of the file with updated name
|
||||
pub fn with_name(&self, new_name: String) -> FileResult<Self> {
|
||||
let new_name = normalize_storage_name(&new_name);
|
||||
if let Err(reason) = validate_storage_name(&new_name) {
|
||||
return Err(FileError::InvalidFileName(format!("{new_name}: {reason}")));
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::domain::services::path_service::{StoragePath, validate_storage_name};
|
||||
use crate::domain::services::path_service::{
|
||||
StoragePath, normalize_storage_name, validate_storage_name,
|
||||
};
|
||||
|
||||
// Re-export entity errors from the centralized module
|
||||
pub use super::entity_errors::{FolderError, FolderResult};
|
||||
@@ -80,6 +82,7 @@ impl Folder {
|
||||
parent_id: Option<String>,
|
||||
owner_id: Option<Uuid>,
|
||||
) -> FolderResult<Self> {
|
||||
let name = normalize_storage_name(&name);
|
||||
// Validate folder name
|
||||
if let Err(reason) = validate_storage_name(&name) {
|
||||
return Err(FolderError::InvalidFolderName(format!("{name}: {reason}")));
|
||||
@@ -171,6 +174,7 @@ impl Folder {
|
||||
modified_at: u64,
|
||||
tree_modified_at: u64,
|
||||
) -> FolderResult<Self> {
|
||||
let name = normalize_storage_name(&name);
|
||||
if let Err(reason) = validate_storage_name(&name) {
|
||||
return Err(FolderError::InvalidFolderName(format!("{name}: {reason}")));
|
||||
}
|
||||
@@ -272,10 +276,13 @@ impl Folder {
|
||||
let storage_path = StoragePath::from_string(&path);
|
||||
|
||||
// Create directly without validation to avoid errors in DTO
|
||||
// conversions. `tree_modified_at` defaults to `modified_at`:
|
||||
// DTO round-trips lose the real rollup signal, so callers
|
||||
// that need a freshly-rolled-up etag must reload from the
|
||||
// conversions. Still NFC-normalize so DTO-reconstructed
|
||||
// entities maintain the storage invariant.
|
||||
// `tree_modified_at` defaults to `modified_at`: DTO
|
||||
// round-trips lose the real rollup signal, so callers that
|
||||
// need a freshly-rolled-up etag must reload from the
|
||||
// repository.
|
||||
let name = normalize_storage_name(&name);
|
||||
Self {
|
||||
id,
|
||||
name,
|
||||
@@ -293,6 +300,7 @@ impl Folder {
|
||||
|
||||
/// Creates a new version of the folder with updated name
|
||||
pub fn with_name(&self, new_name: String) -> FolderResult<Self> {
|
||||
let new_name = normalize_storage_name(&new_name);
|
||||
if let Err(reason) = validate_storage_name(&new_name) {
|
||||
return Err(FolderError::InvalidFolderName(format!(
|
||||
"{new_name}: {reason}"
|
||||
|
||||
@@ -5,6 +5,29 @@
|
||||
//! infrastructure/services/path_service.rs because it has file system dependencies.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
/// NFC-normalize a single file or folder name component.
|
||||
///
|
||||
/// The storage layer (PostgreSQL `storage.files.name` and
|
||||
/// `storage.folders.name`) compares bytes literally — there is no
|
||||
/// Unicode-aware collation in either UNIQUE index. macOS APFS stores
|
||||
/// filenames in NFD (decomposed: `é` = `e` + U+0301), while browsers
|
||||
/// and most other clients post NFC (`é` = U+00E9). Without
|
||||
/// normalization, the same logical filename can land as two distinct
|
||||
/// rows: one from a web upload, one from a NextCloud desktop client
|
||||
/// re-upload of the round-tripped name. The UNIQUE index does not
|
||||
/// catch it because the bytes differ.
|
||||
///
|
||||
/// This function is called at every name-receiving boundary (entity
|
||||
/// constructors, repository path lookups) so the database invariant
|
||||
/// becomes "every stored name is NFC". A one-shot migration
|
||||
/// (`migrate-nfc-filenames`) cleans up rows that pre-date this rule.
|
||||
///
|
||||
/// Pure function — no I/O, allocates one `String`.
|
||||
pub fn normalize_storage_name(name: &str) -> String {
|
||||
name.nfc().collect()
|
||||
}
|
||||
|
||||
/// Validates a single file or folder name component.
|
||||
///
|
||||
@@ -251,4 +274,43 @@ mod tests {
|
||||
assert!(!path.segments().contains(&"..".to_string()));
|
||||
assert!(!path.segments().contains(&".".to_string()));
|
||||
}
|
||||
|
||||
// ── NFC normalization tests ─────────────────────────────────
|
||||
|
||||
/// Plain ASCII names must round-trip identical bytes.
|
||||
#[test]
|
||||
fn test_normalize_ascii_unchanged() {
|
||||
assert_eq!(normalize_storage_name("file.txt"), "file.txt");
|
||||
assert_eq!(normalize_storage_name("My Documents"), "My Documents");
|
||||
}
|
||||
|
||||
/// The macOS APFS / NextCloud-desktop pathological case: `é`
|
||||
/// decomposed as `e` + combining acute (U+0301). Stored bytes
|
||||
/// `65 cc 81` collapse to NFC `c3 a9`.
|
||||
#[test]
|
||||
fn test_normalize_nfd_to_nfc() {
|
||||
let nfd = "caf\u{0065}\u{0301}";
|
||||
let nfc = "caf\u{00E9}";
|
||||
assert_ne!(nfd.as_bytes(), nfc.as_bytes());
|
||||
assert_eq!(normalize_storage_name(nfd), nfc);
|
||||
}
|
||||
|
||||
/// Already-NFC input must round-trip unchanged. This is the
|
||||
/// idempotence property the boundary normalization relies on.
|
||||
#[test]
|
||||
fn test_normalize_nfc_idempotent() {
|
||||
let nfc = "Capture d\u{2019}\u{00E9}cran.png";
|
||||
assert_eq!(normalize_storage_name(nfc), nfc);
|
||||
// And applying twice is the same as once.
|
||||
assert_eq!(normalize_storage_name(&normalize_storage_name(nfc)), nfc);
|
||||
}
|
||||
|
||||
/// Multi-codepoint NFD sequences (combining acute + grave +
|
||||
/// typographic apostrophe) all converge to a single NFC form.
|
||||
#[test]
|
||||
fn test_normalize_mixed_accents() {
|
||||
let nfd = "Capture d\u{2019}\u{0065}\u{0301}cran a\u{0300}.png";
|
||||
let nfc = "Capture d\u{2019}\u{00E9}cran \u{00E0}.png";
|
||||
assert_eq!(normalize_storage_name(nfd), nfc);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user