Edouard Vanbelle
7db547a669
TL;DR:
fix duplicate filename via:
```
docker exec <container> migrate-nfc-filenames --dry-run # preview
docker exec <container> migrate-nfc-filenames # execute
```
== issue ==
Last week I uploaded Capture d'écran 2026-06-03 à 20.04.24.png from the web. It synced down to Nextcloud on my Mac. Two minutes later, the Web UI was showing the file twice.
Both rows had:
- the same name
- the same size
- the same content hash
So why two rows? Because to PostgreSQL, the names weren't the same.
Web upload (browser → Postgres):
"é" stored as 1 codepoint (U+00E9) bytes: c3 a9 ← NFC
NiextCloud client (macOS → Postgres):
"é" stored as 2 codepoints (e + U+0301) bytes: 65 cc 81 ← NFD
macOS's APFS keeps filenames in NFD (decomposed); browsers send NFC (composed). Visually é and é are identical. To WHERE name = $1 they're two different keys. Our UNIQUE index on (folder_id, name, user_id) never fired — and the row count quietly drifted every time a Mac user touched an accented
filename.
== The fix is two halves ==
1. No new duplicates — every name-receiving boundary (file upload, NC PUT, rename, MOVE, path lookup) now NFC-normalizes before touching the database. The storage invariant becomes "every stored name is NFC".
2. Clean up existing data — one-shot migrate-nfc-filenames binary walks storage.files, NFC-normalizes any non-NFC row, and resolves the collisions we've accumulated. Same-content duplicates go to trash (recoverable); different-content collisions get renamed with a .duplicate suffix.
== use of the clean up ==
example of use (do not forget to define env **DATABASE_URL**)
either
`cargo run --bin migrate-nfc-filenames -- --dry-run`
or
`cargo build --bin migrate-nfc-filenames`
`./target/debug/migrate-nfc-filenames --dry-run`
example:
```
% ./target/debug/migrate-nfc-filenames --dry-run
=== NFC filename migration (DRY RUN — no writes) ===
Loaded 543 non-trashed file rows
NORMALIZE 163451b5-5e6c-404b-9b1e-f4b01a2b7269 user=42433185-4717-416d-9a15-4580fff171ec 'Capture d’écran 2026-03-20 à 14.44.50.png' → 'Capture d’écran 2026-03-20 à 14.44.50.png'
NORMALIZE 827dddec-4dd5-48c2-a120-dec5289f7d29 user=969deca6-7935-4f12-a430-4d636b62fa3e 'Capture d’écran 2026-04-03 à 15.43.38.png' → 'Capture d’écran 2026-04-03 à 15.43.38.png'
NORMALIZE 09559934-a620-472d-9ba8-fc3cfeb6dc6f user=a0643a21-0092-4a84-9dde-7ac4e76bc1a5 'Capture d’écran 2026-06-03 à 20.05.38.png' → 'Capture d’écran 2026-06-03 à 20.05.38.png'
NORMALIZE 5ce6dbf9-0562-4758-8783-671aa9069590 user=a0643a21-0092-4a84-9dde-7ac4e76bc1a5 'Capture d’écran 2026-06-05 à 11.07.25.png' → 'Capture d’écran 2026-06-05 à 11.07.25.png'
DEDUP newer=26bcf82b-99cc-45c8-9d69-dd7e5c4484ff (trash, same blob) older=df3adc67-a778-424d-a817-b930c75f3b06 user=a0643a21-0092-4a84-9dde-7ac4e76bc1a5 hash=0d2cc7b0ffce2850
=== Summary ===
scanned : 543
already in NFC : 538
normalized in place (no collision) : 4
dedup-trashed (same content) : 1
renamed to .duplicate : 0
DRY RUN — no rows were written. Re-run without --dry-run to apply.
```
once valid remove --dry-run
119 lines
3.8 KiB
TOML
119 lines
3.8 KiB
TOML
[package]
|
|
name = "oxicloud"
|
|
version = "0.6.0"
|
|
edition = "2024"
|
|
default-run = "oxicloud"
|
|
|
|
|
|
[dependencies]
|
|
mimalloc = { version = "0.1.48", default-features = false }
|
|
axum = { version = "0.8.9", features = ["multipart", "http1", "http2", "tokio", "macros"] }
|
|
tokio = { version = "1.52.0", features = ["rt-multi-thread", "macros", "io-util", "net", "time", "sync", "fs"] }
|
|
tokio-util = { version = "0.7.18", features = ["io", "codec", "compat"] }
|
|
tokio-stream = { version = "0.1.18", features = ["fs"] }
|
|
bytes = "1.11.1"
|
|
tempfile = "3.26.0"
|
|
tower = "0.5.3"
|
|
tower-http = { version = "0.6.8", features = ["fs", "compression-gzip", "compression-br", "trace", "cors", "add-extension", "request-id", "set-header", "limit"] }
|
|
flate2 = "1.1.9"
|
|
tracing = "0.1.44"
|
|
tracing-subscriber = { version = "0.3.22", features = ["env-filter"] }
|
|
chrono = { version = "0.4.44", features = ["serde"] }
|
|
http-body = "1.0.1"
|
|
serde = { version = "1.0.228", features = ["derive"] }
|
|
serde_json = "1.0.149"
|
|
futures = "0.3.32"
|
|
async-stream = "0.3.6"
|
|
async-trait = "0.1.83"
|
|
mime_guess = "2.0.5"
|
|
uuid = { version = "1.23.0", features = ["v4", "v7", "serde"] }
|
|
thiserror = "2.0.18"
|
|
|
|
mockall = { version = "0.14.0", optional = true }
|
|
sqlx = { version = "0.8.6", features = ["postgres", "runtime-tokio", "tls-rustls", "chrono", "uuid", "json", "migrate"] }
|
|
jsonwebtoken = { version = "10.3.0", features = ["rust_crypto"] }
|
|
argon2 = "0.5.3"
|
|
rand_core = { version = "0.6", features = ["std", "getrandom"] }
|
|
quick-xml = "0.39.2"
|
|
dotenvy = "0.15.7"
|
|
moka = { version = "0.12.15", features = ["future", "sync"] }
|
|
http-range-header = "0.4"
|
|
image = { version = "0.25.10", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
|
|
id3 = "1.14"
|
|
mp3-duration = "0.1"
|
|
kamadak-exif = "0.6.1"
|
|
md-5 = "0.11.0"
|
|
sha2 = "0.11.0"
|
|
unicode-normalization = "0.1.24"
|
|
blake3 = { version = "1.8.4", features = ["rayon", "mmap"] }
|
|
hex = "0.4.3"
|
|
http-body-util = "0.1.3"
|
|
percent-encoding = "2.3.2"
|
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots"] }
|
|
base64 = "0.22.1"
|
|
fs2 = "0.4"
|
|
rayon = "1.12.0"
|
|
infer = "0.19"
|
|
async-compression = { version = "0.4.41", features = ["tokio", "gzip"] }
|
|
async_zip = { version = "0.0.18", features = ["tokio", "deflate"] }
|
|
dashmap = "6.1.0"
|
|
socket2 = { version = "0.6.3", features = ["all"] }
|
|
urlencoding = "2.1.3"
|
|
utoipa = { version = "5.4.0", features = ["axum_extras", "uuid", "chrono"] }
|
|
aws-sdk-s3 = "1.129.0"
|
|
aws-config = { version = "1.8.15", features = ["behavior-version-latest"] }
|
|
aws-smithy-types = "1.4.7"
|
|
azure_core = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
azure_storage = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
azure_storage_blobs = { version = "0.21", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
aes-gcm = "0.10.3"
|
|
lru = "0.16.4"
|
|
fastcdc = "4.0.0"
|
|
memmap2 = "0.9.10"
|
|
lettre = { version = "0.11.18", default-features = false, features = ["smtp-transport", "tokio1-rustls-tls", "rustls-native-certs", "builder"] }
|
|
idna = "1.1"
|
|
smol_str = { version = "0.3.2", features = ["serde"] }
|
|
accept-language = "3.1.0"
|
|
askama = "0.16.0"
|
|
|
|
[features]
|
|
default = []
|
|
test_utils = ["mockall"]
|
|
integration_tests = []
|
|
|
|
[lints.rust]
|
|
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(integration_tests)'] }
|
|
|
|
[[bin]]
|
|
name = "generate-openapi"
|
|
path = "src/bin/generate-openapi.rs"
|
|
|
|
[[bin]]
|
|
name = "migrate-nfc-filenames"
|
|
path = "src/bin/migrate-nfc-filenames.rs"
|
|
|
|
[build-dependencies]
|
|
oxc_allocator = "0.125.0"
|
|
oxc_parser = "0.125.0"
|
|
oxc_semantic = "0.125.0"
|
|
oxc_span = "0.125.0"
|
|
oxc_codegen = "0.125.0"
|
|
oxc_minifier = "0.125.0"
|
|
lightningcss = "1.0.0-alpha.71"
|
|
|
|
[profile.release]
|
|
lto = "fat"
|
|
codegen-units = 1
|
|
opt-level = 3
|
|
panic = "abort"
|
|
strip = true
|
|
|
|
[profile.dev]
|
|
opt-level = 1
|
|
debug = true
|
|
|
|
[profile.bench]
|
|
lto = "fat"
|
|
codegen-units = 1
|
|
opt-level = 3
|