From cd3733b459fd7ba8f70bcf958af92b1f52ebc6fe Mon Sep 17 00:00:00 2001 From: Diocrafts Date: Tue, 14 Apr 2026 21:33:38 +0200 Subject: [PATCH] feat: pluggable storage backends (S3, Azure, local) with admin UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement 4-phase external storage backends architecture: Phase 1 - Foundation: - BlobStorageBackend trait (application/ports/blob_storage_ports.rs) - LocalBlobBackend: extracted all tokio::fs ops from DedupService - S3BlobBackend: AWS SDK with custom endpoint support (MinIO, R2, B2) - DedupService refactored to use Arc Phase 2 - Admin Panel: - StorageSettingsService with DB persistence + env override - Storage tab in admin panel (backend selector, S3 form, provider presets) - GET/PUT/POST endpoints for storage settings + connection test - i18n keys (en/es) and BEM CSS Phase 3 - Migration: - MigrationBlobBackend decorator (dual-read: target-first + source fallback) - Background migration job with parallel transfers + progress tracking - Migration UI (progress bar, ETA, pause/resume/verify/complete) - 6 admin API endpoints for migration lifecycle Phase 4 - Enterprise Extras: - CachedBlobBackend: LRU disk cache for remote backends - EncryptedBlobBackend: AES-256-GCM at-rest encryption - AzureBlobBackend: Azure Blob Storage support - RetryBlobBackend: exponential backoff for transient errors - Decorator composition in DI: retry → encryption → cache All 223 tests passing, clippy clean, fmt verified. --- Cargo.lock | 1594 ++++++++++++++++- Cargo.toml | 8 + TODO-STORAGE-BACKENDS.prompt.md | 867 +++++++++ src/application/dtos/settings_dto.rs | 94 + src/application/ports/blob_storage_ports.rs | 88 + src/application/ports/mod.rs | 1 + src/application/services/mod.rs | 1 + .../services/storage_settings_service.rs | 337 ++++ src/common/config.rs | 216 +++ src/common/di.rs | 119 +- .../services/azure_blob_backend.rs | 302 ++++ .../services/cached_blob_backend.rs | 472 +++++ src/infrastructure/services/dedup_service.rs | 291 +-- .../services/encrypted_blob_backend.rs | 298 +++ .../services/local_blob_backend.rs | 277 +++ .../services/migration_blob_backend.rs | 204 +++ src/infrastructure/services/migration_job.rs | 240 +++ src/infrastructure/services/mod.rs | 8 + .../services/retry_blob_backend.rs | 254 +++ .../services/s3_blob_backend.rs | 344 ++++ src/interfaces/api/handlers/admin_handler.rs | 351 +++- static/admin.html | 180 ++ static/css/views/admin.css | 174 ++ static/js/views/admin/admin.js | 368 ++++ static/locales/en.json | 45 +- static/locales/es.json | 45 +- 26 files changed, 6870 insertions(+), 308 deletions(-) create mode 100644 TODO-STORAGE-BACKENDS.prompt.md create mode 100644 src/application/ports/blob_storage_ports.rs create mode 100644 src/application/services/storage_settings_service.rs create mode 100644 src/infrastructure/services/azure_blob_backend.rs create mode 100644 src/infrastructure/services/cached_blob_backend.rs create mode 100644 src/infrastructure/services/encrypted_blob_backend.rs create mode 100644 src/infrastructure/services/local_blob_backend.rs create mode 100644 src/infrastructure/services/migration_blob_backend.rs create mode 100644 src/infrastructure/services/migration_job.rs create mode 100644 src/infrastructure/services/retry_blob_backend.rs create mode 100644 src/infrastructure/services/s3_blob_backend.rs diff --git a/Cargo.lock b/Cargo.lock index 0ed3ad3d..2f35c5df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,53 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + [[package]] name = "adler2" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.7.8" @@ -107,6 +148,17 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + [[package]] name = "async-compression" version = "0.4.37" @@ -126,7 +178,7 @@ version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ - "event-listener", + "event-listener 5.4.1", "event-listener-strategy", "pin-project-lite", ] @@ -153,6 +205,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "async_zip" version = "0.0.18" @@ -161,7 +224,7 @@ checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6" dependencies = [ "async-compression", "crc32fast", - "futures-lite", + "futures-lite 2.6.1", "pin-project", "thiserror 2.0.18", "tokio", @@ -189,6 +252,476 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "1.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.5", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.3.0", + "hex", + "http 1.4.0", + "sha1", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http 0.63.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand 2.3.0", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d65fddc3844f902dfe1864acb8494db5f9342015ee3ab7890270d36fbd2e01c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http 0.62.6", + "aws-smithy-json 0.61.9", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand 2.3.0", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "lru", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.97.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.5", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.3.0", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.99.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.5", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.3.0", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.101.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.5", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand 2.3.0", + "http 0.2.12", + "http 1.4.0", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http 0.63.6", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "p256 0.11.1", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.63.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87294a084b43d649d967efe58aa1f9e0adc260e13a6938eb904c0ae9b45824ae" +dependencies = [ + "aws-smithy-http 0.62.6", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf09d74e5e32f76b8762da505a3cd59303e367a664ca67295387baa8c1d7548" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.62.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.8.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand 2.3.0", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" +dependencies = [ + "base64-simd 0.8.0", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.8.8" @@ -200,10 +733,10 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "itoa", "matchit", @@ -232,8 +765,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -254,12 +787,110 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "azure_core" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "dyn-clone", + "futures", + "getrandom 0.2.17", + "hmac", + "http-types", + "once_cell", + "paste", + "pin-project", + "quick-xml 0.31.0", + "rand 0.8.5", + "reqwest", + "rustc_version", + "serde", + "serde_json", + "sha2", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_storage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +dependencies = [ + "RustyXML", + "async-lock", + "async-trait", + "azure_core", + "bytes", + "serde", + "serde_derive", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +dependencies = [ + "RustyXML", + "azure_core", + "azure_storage", + "azure_svc_blobstorage", + "bytes", + "futures", + "serde", + "serde_derive", + "serde_json", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_svc_blobstorage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" +dependencies = [ + "azure_core", + "bytes", + "futures", + "log", + "once_cell", + "serde", + "serde_json", + "time", +] + +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base16ct" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -419,6 +1050,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "castaway" version = "0.2.4" @@ -435,6 +1076,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -475,6 +1118,25 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cobs" version = "0.3.0" @@ -572,6 +1234,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -608,6 +1280,19 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc-fast" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3" +dependencies = [ + "crc", + "digest", + "rand 0.9.2", + "regex", + "rustversion", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -660,6 +1345,18 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -679,6 +1376,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -714,6 +1412,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -783,6 +1490,16 @@ dependencies = [ "matches", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.10" @@ -801,6 +1518,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc3dc5ad92c2e2d1c193bbbbdf2ea477cb81331de4f3103f267ca18368b988c4" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -859,18 +1577,42 @@ dependencies = [ "dtoa", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -879,8 +1621,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -906,23 +1648,43 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff 0.12.1", + "generic-array", + "group 0.12.1", + "pkcs8 0.9.0", + "rand_core 0.6.4", + "sec1 0.3.0", + "subtle", + "zeroize", +] + [[package]] name = "elliptic-curve" version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest", - "ff", + "ff 0.13.1", "generic-array", - "group", + "group 0.13.0", "hkdf", "pem-rfc7468", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "subtle", "zeroize", ] @@ -975,6 +1737,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "event-listener" version = "5.4.1" @@ -992,10 +1760,19 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener", + "event-listener 5.4.1", "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1011,6 +1788,16 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "ff" version = "0.13.1" @@ -1066,6 +1853,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1091,6 +1893,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -1156,13 +1964,28 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-lite" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" dependencies = [ - "fastrand", + "fastrand 2.3.0", "futures-core", "futures-io", "parking", @@ -1220,6 +2043,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1229,7 +2063,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -1260,6 +2094,16 @@ dependencies = [ "wasip3", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gif" version = "0.14.1" @@ -1270,17 +2114,47 @@ dependencies = [ "weezl", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff 0.12.1", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.13" @@ -1292,7 +2166,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.0", "indexmap", "slab", "tokio", @@ -1383,6 +2257,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.0" @@ -1393,6 +2278,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1400,7 +2296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.0", ] [[package]] @@ -1411,8 +2307,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1422,6 +2318,26 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel", + "base64 0.13.1", + "futures-lite 1.13.0", + "infer 0.2.3", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url", +] + [[package]] name = "httparse" version = "1.10.1" @@ -1434,6 +2350,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.8.1" @@ -1444,9 +2384,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", - "http", - "http-body", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1457,41 +2397,73 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", - "hyper", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", - "rustls", + "rustls 0.23.36", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", "webpki-roots 1.0.5", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "tokio", "tower-service", "tracing", @@ -1680,6 +2652,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + [[package]] name = "infer" version = "0.19.0" @@ -1689,6 +2667,24 @@ dependencies = [ "cfb", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1729,6 +2725,16 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.85" @@ -1751,12 +2757,12 @@ version = "10.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" dependencies = [ - "base64", + "base64 0.22.1", "ed25519-dalek", "getrandom 0.2.17", "hmac", "js-sys", - "p256", + "p256 0.13.2", "p384", "pem", "rand 0.8.5", @@ -1764,7 +2770,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "signature", + "signature 2.2.0", "simple_asn1", ] @@ -1902,6 +2908,15 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1996,7 +3011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -2037,7 +3052,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "equivalent", - "event-listener", + "event-listener 5.4.1", "futures-util", "parking_lot", "portable-atomic", @@ -2074,7 +3089,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 1.4.0", "httparse", "memchr", "mime", @@ -2088,6 +3103,23 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nonmax" version = "0.5.5" @@ -2171,6 +3203,56 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openssl" +version = "0.10.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2f2c0eba47118757e4c6d2bff2838f3e0523380021356e7875e858372ce644" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "outref" version = "0.1.0" @@ -2549,12 +3631,19 @@ dependencies = [ name = "oxicloud" version = "0.5.5" dependencies = [ + "aes-gcm", "argon2", "async-compression", "async-stream", "async_zip", + "aws-config", + "aws-sdk-s3", + "aws-smithy-types", "axum", - "base64", + "azure_core", + "azure_storage", + "azure_storage_blobs", + "base64 0.22.1", "blake3", "bytes", "chrono", @@ -2564,15 +3653,16 @@ dependencies = [ "fs2", "futures", "hex", - "http-body", + "http-body 1.0.1", "http-body-util", "http-range-header", "id3", "image", - "infer", + "infer 0.19.0", "jsonwebtoken", "kamadak-exif", "lightningcss", + "lru", "md-5", "mimalloc", "mime_guess", @@ -2585,14 +3675,14 @@ dependencies = [ "oxc_parser", "oxc_span", "percent-encoding", - "quick-xml", + "quick-xml 0.39.2", "rand_core 0.6.4", "rayon", "reqwest", "serde", "serde_json", "sha2", - "socket2", + "socket2 0.6.2", "sqlx", "tempfile", "thiserror 2.0.18", @@ -2608,14 +3698,25 @@ dependencies = [ "uuid", ] +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", + "sha2", +] + [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2", ] @@ -2626,8 +3727,8 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", "primeorder", "sha2", ] @@ -2702,6 +3803,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pastey" version = "0.1.1" @@ -2720,7 +3827,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -2786,7 +3893,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" dependencies = [ - "fastrand", + "fastrand 2.3.0", "phf_shared 0.13.1", ] @@ -2872,9 +3979,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -2883,8 +4000,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", ] [[package]] @@ -2906,6 +4023,18 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -2996,7 +4125,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -3043,6 +4172,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.39.2" @@ -3064,8 +4203,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", - "socket2", + "rustls 0.23.36", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -3084,7 +4223,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash", - "rustls", + "rustls 0.23.36", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -3102,7 +4241,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] @@ -3128,6 +4267,19 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -3149,6 +4301,16 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3169,6 +4331,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -3187,6 +4358,15 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rayon" version = "1.11.0" @@ -3248,6 +4428,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.8" @@ -3269,38 +4455,55 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", - "http", - "http-body", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.8.1", + "hyper-rustls 0.27.7", + "hyper-tls", "hyper-util", "js-sys", "log", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.36", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-native-tls", + "tokio-rustls 0.26.4", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots 1.0.5", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -3366,10 +4569,10 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] @@ -3402,20 +4605,45 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.9", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -3426,12 +4654,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3449,32 +4688,88 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct 0.1.1", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "sec1" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "self_cell" version = "1.2.2" @@ -3556,6 +4851,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror 1.0.69", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3605,6 +4911,26 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "signature" version = "2.2.0" @@ -3675,6 +5001,16 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -3694,6 +5030,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.3" @@ -3701,7 +5047,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] [[package]] @@ -3723,13 +5069,13 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "crc", "crossbeam-queue", "either", - "event-listener", + "event-listener 5.4.1", "futures-core", "futures-intrusive", "futures-io", @@ -3741,7 +5087,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls", + "rustls 0.23.36", "serde", "serde_json", "sha2", @@ -3800,7 +5146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "bytes", @@ -3844,7 +5190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "chrono", @@ -3991,7 +5337,7 @@ version = "3.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ - "fastrand", + "fastrand 2.3.0", "getrandom 0.4.1", "once_cell", "rustix", @@ -4072,6 +5418,7 @@ checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", + "js-sys", "num-conv", "powerfmt", "serde_core", @@ -4130,7 +5477,8 @@ dependencies = [ "libc", "mio", "pin-project-lite", - "socket2", + "signal-hook-registry", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] @@ -4146,13 +5494,33 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.36", "tokio", ] @@ -4208,8 +5576,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "http-range-header", "httpdate", @@ -4376,6 +5744,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -4392,6 +5770,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -4473,6 +5852,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + [[package]] name = "want" version = "0.3.1" @@ -4482,6 +5867,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -4593,6 +5984,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -5065,6 +6469,12 @@ dependencies = [ "tap", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index fccc9be6..591f331f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,14 @@ dashmap = "6" socket2 = { version = "0.6.2", features = ["all"] } urlencoding = "2.1.3" utoipa = { version = "5", features = ["axum_extras", "uuid", "chrono"] } +aws-sdk-s3 = "1" +aws-config = { version = "1", features = ["behavior-version-latest"] } +aws-smithy-types = "1" +azure_core = "0.21" +azure_storage = "0.21" +azure_storage_blobs = "0.21" +aes-gcm = "0.10" +lru = "0.12" [features] default = [] diff --git a/TODO-STORAGE-BACKENDS.prompt.md b/TODO-STORAGE-BACKENDS.prompt.md new file mode 100644 index 00000000..6b936ded --- /dev/null +++ b/TODO-STORAGE-BACKENDS.prompt.md @@ -0,0 +1,867 @@ +# External Storage Backends — Implementation Plan + +> **Purpose**: This prompt provides Claude Code with full architectural context to implement pluggable blob storage backends (S3, Backblaze B2, MinIO, etc.) for OxiCloud, across 4 phases. All changes MUST respect the existing hexagonal architecture, BLAKE3 dedup system, and coding conventions defined in `CLAUDE.md`. + +--- + +## Current Architecture Summary + +### How blobs are stored today + +- **Content-addressable**: Files hashed with BLAKE3 → stored at `.blobs/{2-char-prefix}/{hash}.blob` +- **Dedup index**: PostgreSQL `storage.blobs` table (hash PK, ref_count, size, content_type) +- **Write-first strategy**: Blob written to disk BEFORE PostgreSQL upsert (PG connection never held during disk I/O) +- **Streaming reads**: 256 KB chunks via `tokio::fs::File` + `ReaderStream` +- **Range support**: `AsyncSeekExt::seek()` + `file.take()` for HTTP Range requests + +### Key files + +| File | Role | +|------|------| +| `src/application/ports/dedup_ports.rs` | `DedupPort` trait — 12 methods, the hexagonal port | +| `src/infrastructure/services/dedup_service.rs` | `DedupService` struct — sole implementation of `DedupPort` | +| `src/common/di.rs` | `AppServiceFactory` — DI composition root, builds `DedupService` | +| `src/common/config.rs` | `StorageConfig`, `AppConfig` — env var loading | +| `src/interfaces/api/handlers/admin_handler.rs` | Admin API handlers (OIDC pattern to follow) | +| `src/application/services/admin_settings_service.rs` | `AdminSettingsService` — runtime settings with env override | +| `src/domain/repositories/settings_repository.rs` | `SettingsRepository` trait | +| `src/infrastructure/repositories/pg/settings_pg_repository.rs` | PostgreSQL settings impl | +| `static/admin.html` | Admin panel HTML (3 tabs: Dashboard, Users, OIDC) | +| `static/js/views/admin/admin.js` | Admin panel JS logic | + +### DedupService filesystem operations (candidates for extraction) + +These are the EXACT `tokio::fs` calls inside `DedupService` that must be delegated to the new `BlobStorageBackend` trait: + +``` +initialize() → fs::create_dir_all (blob_root, temp_root, 256 prefix dirs) +store_from_file() → fs::metadata, fs::try_exists, fs::rename, fs::copy, fs::remove_file +read_blob_stream() → File::open + ReaderStream +read_blob_range_stream() → File::open + seek + take + ReaderStream +blob_size() → fs::metadata +remove_reference() → fs::remove_file (after PG commit) +verify_integrity() → spawn_blocking with path checks +blob_path() → PathBuf computation (sync) +``` + +### DedupService PostgreSQL operations (stay in DedupService, untouched) + +``` +store_from_file() → INSERT … ON CONFLICT … RETURNING ref_count +blob_exists() → SELECT EXISTS from storage.blobs +get_blob_metadata() → SELECT from storage.blobs +add_reference() → UPDATE ref_count + 1 +remove_reference() → BEGIN TX → SELECT FOR UPDATE → DELETE if ref_count=0 → COMMIT +get_stats() → SELECT COUNT, SUM from storage.blobs +verify_integrity() → Streaming cursor SELECT from storage.blobs +``` + +--- + +## Phase 1 — Foundation (Backend Trait + Local + S3) + +### Task 1.1: Create `BlobStorageBackend` trait + +**File**: `src/application/ports/blob_storage_ports.rs` (NEW) + +Create a minimal trait that abstracts ONLY raw byte I/O operations: + +```rust +use async_trait::async_trait; +use bytes::Bytes; +use futures::Stream; +use std::path::Path; +use std::pin::Pin; +use crate::domain::errors::DomainError; + +/// Health check result for storage backend connectivity +#[derive(Debug, Clone, serde::Serialize)] +pub struct StorageHealthStatus { + pub connected: bool, + pub backend_type: String, + pub message: String, + /// Optional: available space in bytes (if backend reports it) + pub available_bytes: Option, +} + +/// Minimal trait for blob byte I/O — decoupled from dedup logic. +/// +/// Implementations: `LocalBlobBackend`, `S3BlobBackend`, etc. +/// DedupService owns an `Arc` and delegates +/// all filesystem/object-store operations through this trait. +#[async_trait] +pub trait BlobStorageBackend: Send + Sync + 'static { + /// Initialize the backend (create directories, verify bucket access, etc.) + async fn initialize(&self) -> Result<(), DomainError>; + + /// Store a blob from a local temporary file. + /// The backend MUST handle the case where the blob already exists (idempotent). + /// Returns the number of bytes stored. + async fn put_blob(&self, hash: &str, source_path: &Path) -> Result; + + /// Stream the full blob content. + async fn get_blob_stream( + &self, + hash: &str, + ) -> Result> + Send>>, DomainError>; + + /// Stream a byte range of the blob (for HTTP Range requests). + async fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> Result> + Send>>, DomainError>; + + /// Delete a blob by hash. Must be idempotent (no error if already deleted). + async fn delete_blob(&self, hash: &str) -> Result<(), DomainError>; + + /// Check if a blob exists in the backend. + async fn blob_exists(&self, hash: &str) -> Result; + + /// Get blob size in bytes without downloading content. + async fn blob_size(&self, hash: &str) -> Result; + + /// Verify connectivity and permissions. Used by admin "Test Connection" button. + async fn health_check(&self) -> Result; + + /// Return the backend type name (for display in admin panel). + fn backend_type(&self) -> &'static str; +} +``` + +Register in `src/application/ports/mod.rs` and `src/application/mod.rs`. + +### Task 1.2: Create `LocalBlobBackend` + +**File**: `src/infrastructure/services/local_blob_backend.rs` (NEW) + +Extract ALL `tokio::fs` operations from `DedupService` into this struct. This is a **pure refactor** — zero behavior change. + +```rust +pub struct LocalBlobBackend { + blob_root: PathBuf, + temp_root: PathBuf, +} +``` + +Methods to implement from the trait, mapping from current DedupService code: + +| Trait method | Source in DedupService | Key logic | +|---|---|---| +| `initialize()` | `DedupService::initialize()` lines 115-154 | Create `.blobs/`, `.dedup_temp/`, 256 prefix dirs | +| `put_blob()` | `DedupService::store_from_file()` lines 196-253 | `fs::try_exists` → `fs::rename` (EXDEV fallback `fs::copy`) → cleanup source | +| `get_blob_stream()` | `DedupService::read_blob_stream()` lines 306-323 | `File::open` → `ReaderStream::with_capacity(256KB)` | +| `get_blob_range_stream()` | `DedupService::read_blob_range_stream()` lines 325-353 | `File::open` → `seek` → `take` → `ReaderStream` | +| `delete_blob()` | `DedupService::remove_reference()` line ~469 | `fs::remove_file` | +| `blob_exists()` | New (was inline `fs::try_exists`) | `fs::try_exists(blob_path)` | +| `blob_size()` | `DedupService::blob_size()` lines 355-369 | `fs::metadata().len()` | +| `health_check()` | New | Check blob_root is writable, return disk available via `statvfs` | +| `backend_type()` | New | Return `"local"` | + +Also add a public helper: +```rust +pub fn blob_path(&self, hash: &str) -> PathBuf { + let prefix = &hash[..2]; + self.blob_root.join(prefix).join(format!("{hash}.blob")) +} +``` + +Register in `src/infrastructure/services/mod.rs`. + +### Task 1.3: Refactor `DedupService` to use `BlobStorageBackend` + +**File**: `src/infrastructure/services/dedup_service.rs` (MODIFY) + +Changes: +1. Add field: `backend: Arc` +2. Remove fields: `blob_root: PathBuf`, `temp_root: PathBuf` (moved to `LocalBlobBackend`) +3. Update constructor to accept `Arc` instead of `storage_root: &Path` +4. Replace all direct `tokio::fs` calls with `self.backend.*` calls +5. Keep `blob_path()` in DedupPort as a delegation: `self.backend.blob_path()` — BUT since `blob_path()` returns `PathBuf` and is used by thumbnails/caching services, consider adding it to the backend trait OR keeping a separate method. For S3 backends, this should return a virtual path or the method should be deprecated in favor of streaming. + +**Critical**: `hash_file()` stays in `DedupService` (BLAKE3 hashing is NOT a backend concern — it always runs on local temp files before upload). + +**Critical**: The write-first strategy is preserved: +``` +1. hash_file() on local temp file +2. self.backend.put_blob(hash, temp_path) ← backend moves/uploads +3. INSERT INTO storage.blobs … ON CONFLICT ← PostgreSQL upsert +``` + +**Critical**: `remove_reference()` flow preserved: +``` +1. BEGIN TX → SELECT FOR UPDATE → check ref_count +2. If ref_count == 1 → DELETE FROM storage.blobs → COMMIT +3. self.backend.delete_blob(hash) ← after PG commit +``` + +### Task 1.4: Create `S3BlobBackend` + +**File**: `src/infrastructure/services/s3_blob_backend.rs` (NEW) + +**Dependency to add to `Cargo.toml`**: +```toml +aws-sdk-s3 = "1" +aws-config = { version = "1", features = ["behavior-version-latest"] } +aws-smithy-types = "1" # For ByteStream +``` + +> Note: `aws-sdk-s3` is the official AWS SDK for Rust. It's compatible with ALL S3-compatible services (Backblaze B2, MinIO, Cloudflare R2, DigitalOcean Spaces, Wasabi) via custom endpoint configuration. + +```rust +pub struct S3BlobBackend { + client: aws_sdk_s3::Client, + bucket: String, +} +``` + +**S3 key scheme**: Same as local — `{2-char-prefix}/{hash}.blob` (e.g., `a3/a3c5f2e8d1…blob`) + +Method mapping: + +| Trait method | S3 operation | +|---|---| +| `initialize()` | `head_bucket()` to verify bucket exists + permissions | +| `put_blob()` | `put_object()` with `Body::from_path(source_path)`. Check existence with `head_object()` first for idempotency | +| `get_blob_stream()` | `get_object()` → `.body.into_async_read()` → `ReaderStream` | +| `get_blob_range_stream()` | `get_object().range(format!("bytes={start}-{end}"))` → stream | +| `delete_blob()` | `delete_object()` (already idempotent in S3) | +| `blob_exists()` | `head_object()` — 200 = true, 404 = false | +| `blob_size()` | `head_object()` → `.content_length()` | +| `health_check()` | `head_bucket()` + `list_objects_v2(max_keys=1)` | +| `backend_type()` | Return `"s3"` | + +**S3 Client construction**: Must support custom endpoints for non-AWS providers: + +```rust +impl S3BlobBackend { + pub async fn new(config: &S3StorageConfig) -> Result { + let mut s3_config_builder = aws_sdk_s3::config::Builder::new() + .region(aws_sdk_s3::config::Region::new(config.region.clone())) + .credentials_provider( + aws_sdk_s3::config::Credentials::new( + &config.access_key, + &config.secret_key, + None, None, "oxicloud", + ) + ) + .behavior_version_latest(); + + if let Some(endpoint) = &config.endpoint_url { + s3_config_builder = s3_config_builder + .endpoint_url(endpoint) + .force_path_style(config.force_path_style); + } + + let client = aws_sdk_s3::Client::from_conf(s3_config_builder.build()); + Ok(Self { client, bucket: config.bucket.clone() }) + } +} +``` + +### Task 1.5: Add storage backend configuration + +**File**: `src/common/config.rs` (MODIFY) + +Add to existing `StorageConfig`: + +```rust +#[derive(Debug, Clone)] +pub enum StorageBackendType { + Local, + S3, +} + +#[derive(Debug, Clone)] +pub struct S3StorageConfig { + pub endpoint_url: Option, // OXICLOUD_S3_ENDPOINT_URL + pub bucket: String, // OXICLOUD_S3_BUCKET + pub region: String, // OXICLOUD_S3_REGION (default: "us-east-1") + pub access_key: String, // OXICLOUD_S3_ACCESS_KEY + pub secret_key: String, // OXICLOUD_S3_SECRET_KEY + pub force_path_style: bool, // OXICLOUD_S3_FORCE_PATH_STYLE (default: false) +} + +// Add to existing StorageConfig: +pub struct StorageConfig { + pub root_dir: String, // existing + pub chunk_size: usize, // existing + pub parallel_threshold: usize, // existing + pub trash_retention_days: u32, // existing + pub max_upload_size: usize, // existing + pub backend: StorageBackendType, // NEW — OXICLOUD_STORAGE_BACKEND (default: "local") + pub s3: Option, // NEW — populated when backend=s3 +} +``` + +**Env var loading** in `AppConfig::from_env()`: +``` +OXICLOUD_STORAGE_BACKEND → "local" | "s3" (default: "local") +OXICLOUD_S3_ENDPOINT_URL → Optional custom endpoint +OXICLOUD_S3_BUCKET → Required when backend=s3 +OXICLOUD_S3_REGION → Default "us-east-1" +OXICLOUD_S3_ACCESS_KEY → Required when backend=s3 +OXICLOUD_S3_SECRET_KEY → Required when backend=s3 +OXICLOUD_S3_FORCE_PATH_STYLE → Default false +``` + +### Task 1.6: Wire backend selection in DI + +**File**: `src/common/di.rs` (MODIFY) + +In `create_core_services()`, replace the current DedupService construction: + +```rust +// Build storage backend based on config +let blob_backend: Arc = match self.config.storage.backend { + StorageBackendType::Local => { + Arc::new(LocalBlobBackend::new(&self.storage_path)) + } + StorageBackendType::S3 => { + let s3_config = self.config.storage.s3.as_ref() + .expect("S3 config required when backend=s3"); + Arc::new(S3BlobBackend::new(s3_config).await?) + } +}; + +blob_backend.initialize().await?; + +let dedup_service = Arc::new(DedupService::new( + blob_backend.clone(), + db_pool.clone(), + maintenance_pool.clone(), +)); +``` + +### Task 1.7: Handle `blob_path()` deprecation path + +The `DedupPort::blob_path()` method returns a `PathBuf` and is used by: +- Thumbnail generation service (needs local file access) +- File content caching (moka cache) + +For S3 backends, `blob_path()` has no meaning. Solutions: +1. **For thumbnails**: Change thumbnail service to accept a `Stream` instead of a `PathBuf`, OR download to a temp file first +2. **For caching**: Cache already works with streams +3. Keep `blob_path()` on `DedupPort` but make it return `Option` (None for remote backends) — consumers must handle the None case + +Search for all callers of `blob_path()` and update them. + +### Task 1.8: Tests + +- Unit test `LocalBlobBackend` in isolation (mock filesystem with temp dirs) +- Unit test `S3BlobBackend` with a mock S3 (use `aws-smithy-runtime` test utilities or `mockall`) +- Integration test: `DedupService` with `LocalBlobBackend` must pass ALL existing tests unchanged (this proves the refactor is correct) +- Add `#[cfg(test)]` inline tests in each new file following existing project convention + +### Task 1.9: Pre-commit validation + +```bash +cargo fmt --all +cargo clippy -- -D warnings +cargo test --workspace +``` + +ALL existing ~208 tests MUST pass. Zero regressions. + +--- + +## Phase 2 — Admin Panel Configuration + +### Task 2.1: Storage settings service + +**File**: `src/application/services/storage_settings_service.rs` (NEW) + +Follow the EXACT same pattern as `AdminSettingsService` for OIDC. Create `StorageSettingsService`: + +```rust +pub struct StorageSettingsService { + settings_repo: Arc, + env_storage_config: StorageConfig, // from AppConfig at startup +} +``` + +**Methods** (follow OIDC pattern): + +| Method | Purpose | +|---|---| +| `get_storage_settings()` | Load from DB (category: `"storage"`), mask secrets, mark env overrides | +| `save_storage_settings(dto, user_id)` | Upsert each field to `admin_settings`, mark secrets with `is_secret: true` | +| `test_storage_connection(dto)` | Build temporary backend from DTO config, call `health_check()`, return result | +| `load_effective_storage_config()` | Merge: DB settings + env var overrides + defaults | +| `get_env_overrides()` | Return list of `OXICLOUD_S3_*` / `OXICLOUD_STORAGE_*` env vars that are set | + +**DB keys** (category: `"storage"`): + +``` +storage.backend → "local" | "s3" +storage.s3.endpoint_url → string (optional) +storage.s3.bucket → string +storage.s3.region → string +storage.s3.access_key → string (is_secret: true) +storage.s3.secret_key → string (is_secret: true) +storage.s3.force_path_style → "true" | "false" +``` + +### Task 2.2: Storage admin API endpoints + +**File**: `src/interfaces/api/handlers/admin_handler.rs` (MODIFY) + +Add to `admin_routes()`: + +```rust +.route("/settings/storage", get(get_storage_settings)) +.route("/settings/storage", put(save_storage_settings)) +.route("/settings/storage/test", post(test_storage_connection)) +``` + +**Handler implementations** (follow OIDC handlers exactly): + +| Handler | Method | Body | Response | +|---|---|---|---| +| `get_storage_settings` | GET | — | `StorageSettingsDto` (secrets masked, env_overrides listed) | +| `save_storage_settings` | PUT | `SaveStorageSettingsDto` | `{ "message": "Storage settings saved" }` | +| `test_storage_connection` | POST | `TestStorageConnectionDto` | `StorageTestResultDto { connected, message, available_bytes }` | + +**DTOs** (add to `src/application/dtos/`): + +```rust +#[derive(Serialize)] +pub struct StorageSettingsDto { + pub backend: String, // "local" | "s3" + pub s3_endpoint_url: Option, + pub s3_bucket: Option, + pub s3_region: Option, + pub s3_access_key_set: bool, // masked — never send actual key + pub s3_secret_key_set: bool, // masked — never send actual secret + pub s3_force_path_style: bool, + pub env_overrides: Vec, // which fields are locked by env vars + // Current stats + pub current_backend: String, + pub total_blobs: u64, + pub total_bytes_stored: u64, + pub dedup_ratio: f64, +} + +#[derive(Deserialize)] +pub struct SaveStorageSettingsDto { + pub backend: String, + pub s3_endpoint_url: Option, + pub s3_bucket: Option, + pub s3_region: Option, + pub s3_access_key: Option, // only sent if changed + pub s3_secret_key: Option, // only sent if changed + pub s3_force_path_style: Option, +} + +#[derive(Deserialize)] +pub struct TestStorageConnectionDto { + pub backend: String, + pub s3_endpoint_url: Option, + pub s3_bucket: Option, + pub s3_region: Option, + pub s3_access_key: Option, + pub s3_secret_key: Option, + pub s3_force_path_style: Option, +} + +#[derive(Serialize)] +pub struct StorageTestResultDto { + pub connected: bool, + pub message: String, + pub backend_type: String, + pub available_bytes: Option, +} +``` + +### Task 2.3: Wire `StorageSettingsService` into DI + +**File**: `src/common/di.rs` (MODIFY) + +Add `StorageSettingsService` construction alongside `AdminSettingsService`. Add to `AppState`. + +### Task 2.4: Admin Panel — Storage tab (HTML) + +**File**: `static/admin.html` (MODIFY) + +Add a 4th tab button after the OIDC tab: + +```html + +``` + +Add `tab-storage` content div with: + +1. **Backend selector** — Radio buttons: Local Filesystem / S3-Compatible +2. **S3 configuration form** (shown/hidden based on selector): + - Provider preset dropdown (Amazon S3, Backblaze B2, Cloudflare R2, MinIO, DigitalOcean Spaces, Wasabi, Custom) + - Endpoint URL field + - Bucket field + - Region field + - Access Key ID field + - Secret Key field (password type) + - Force Path Style checkbox + - ENV badges on fields overridden by env vars (same as OIDC) +3. **Test Connection button** → calls `POST /api/admin/settings/storage/test` +4. **Save button** → calls `PUT /api/admin/settings/storage` +5. **Current Status section**: + - Active backend type + - Total blobs / total size / dedup ratio (from `DedupStatsDto`) +6. **Migration section** (Phase 3 — can be placeholder with "Coming soon") + +### Task 2.5: Admin Panel — Storage tab (JS) + +**File**: `static/js/views/admin/admin.js` (MODIFY) + +Follow OIDC tab patterns: + +```javascript +// Provider presets +const STORAGE_PRESETS = { + 'aws': { endpoint: '', region: 'us-east-1', pathStyle: false }, + 'backblaze': { endpoint: 's3.{region}.backblazeb2.com', region: 'us-west-004', pathStyle: false }, + 'cloudflare-r2': { endpoint: '{accountId}.r2.cloudflarestorage.com', region: 'auto', pathStyle: true }, + 'minio': { endpoint: 'http://localhost:9000', region: 'us-east-1', pathStyle: true }, + 'digitalocean': { endpoint: '{region}.digitaloceanspaces.com', region: 'nyc3', pathStyle: false }, + 'wasabi': { endpoint: 's3.{region}.wasabisys.com', region: 'us-east-1', pathStyle: false }, + 'custom': { endpoint: '', region: '', pathStyle: false }, +}; +``` + +Functions: +- `loadStorage()` — `GET /api/admin/settings/storage` → populate form +- `saveStorage()` — Collect form → `PUT /api/admin/settings/storage` +- `testStorageConnection()` — Collect form → `POST /api/admin/settings/storage/test` → show result +- `onPresetChange(preset)` — Auto-fill endpoint/region/pathStyle from preset +- `toggleS3Form(visible)` — Show/hide S3 fields when backend radio changes + +### Task 2.6: i18n keys + +**Files**: `static/locales/*.json` (MODIFY at minimum `en.json` and `es.json`) + +Add translation keys for all new UI labels: +``` +admin.tab_storage +admin.storage_backend +admin.storage_local +admin.storage_s3 +admin.storage_provider_preset +admin.storage_endpoint_url +admin.storage_bucket +admin.storage_region +admin.storage_access_key +admin.storage_secret_key +admin.storage_path_style +admin.storage_test_connection +admin.storage_test_success +admin.storage_test_failure +admin.storage_save +admin.storage_saved +admin.storage_current_status +admin.storage_total_blobs +admin.storage_total_size +admin.storage_dedup_ratio +admin.storage_migration +admin.storage_migration_coming_soon +``` + +### Task 2.7: CSS for storage tab + +**File**: `static/css/admin.css` (MODIFY) + +Add styles for: +- `.storage-backend-selector` — Radio button group +- `.storage-provider-presets` — Dropdown styling +- `.storage-form` — Form fields (reuse existing OIDC form patterns) +- `.storage-status-grid` — Stats display +- BEM methodology, CSS custom properties for colors, no raw hex/rgb + +--- + +## Phase 3 — Migration Between Backends + +### Task 3.1: `MigrationBlobBackend` wrapper + +**File**: `src/infrastructure/services/migration_blob_backend.rs` (NEW) + +A `BlobStorageBackend` decorator that enables zero-downtime migration between backends: + +```rust +pub struct MigrationBlobBackend { + source: Arc, // old backend (read fallback) + target: Arc, // new backend (primary for writes) + state: Arc>, +} + +pub struct MigrationState { + pub status: MigrationStatus, + pub total_blobs: u64, + pub migrated_blobs: u64, + pub migrated_bytes: u64, + pub failed_blobs: Vec, // hashes that failed + pub started_at: Option>, + pub completed_at: Option>, +} + +pub enum MigrationStatus { + Idle, // no migration in progress + Running, // background job active + Paused, // manually paused + Completed, // all blobs migrated + Failed, // unrecoverable error +} +``` + +**Behavior**: + +| Operation | During Migration | +|---|---| +| `put_blob()` | Write to **target** only | +| `get_blob_stream()` | Try **target** first → fallback to **source** (+ schedule lazy copy) | +| `get_blob_range_stream()` | Same fallback strategy | +| `delete_blob()` | Delete from **both** (best-effort on source) | +| `blob_exists()` | Check **target** first, then **source** | +| `blob_size()` | Check **target** first, then **source** | + +### Task 3.2: Background migration job + +**File**: `src/infrastructure/services/migration_job.rs` (NEW) + +```rust +pub async fn run_migration( + source: Arc, + target: Arc, + pool: Arc, + state: Arc>, + concurrency: usize, // default: 4 parallel transfers + bandwidth_limit: Option, // bytes/sec, None = unlimited +) -> Result<(), DomainError> +``` + +**Algorithm**: +1. Query `SELECT hash, size FROM storage.blobs ORDER BY hash` with streaming cursor +2. For each blob hash: + a. Check if already exists in target (`target.blob_exists(hash)`) + b. If not: stream from source → write to temp file → `target.put_blob(hash, temp)` + c. Update `MigrationState` counters + d. Respect bandwidth limit via `tokio::time::sleep` throttling +3. Use `futures::stream::buffer_unordered(concurrency)` for parallel transfers +4. On error: log, add to `failed_blobs`, continue (don't abort entire migration) + +### Task 3.3: Migration API endpoints + +**File**: `src/interfaces/api/handlers/admin_handler.rs` (MODIFY) + +Add to `admin_routes()`: + +```rust +.route("/storage/migration", get(get_migration_status)) +.route("/storage/migration/start", post(start_migration)) +.route("/storage/migration/pause", post(pause_migration)) +.route("/storage/migration/resume", post(resume_migration)) +.route("/storage/migration/complete", post(complete_migration)) +``` + +| Endpoint | Purpose | +|---|---| +| `GET /migration` | Return `MigrationState` (status, progress, ETA) | +| `POST /migration/start` | Begin background migration from current → configured backend | +| `POST /migration/pause` | Pause the background job | +| `POST /migration/resume` | Resume paused migration | +| `POST /migration/complete` | Finalize: switch primary backend, optionally clean up source | + +### Task 3.4: Migration UI in admin panel + +In the Storage tab's Migration section: + +1. **Start Migration button** (when backend config differs from active) +2. **Progress bar**: `{migrated} / {total} blobs ({percent}%) — {bytes_migrated} transferred` +3. **Estimated time remaining** (based on throughput) +4. **Pause / Resume button** +5. **Complete Migration button** (enabled only when 100% migrated) +6. **Failed blobs list** (expandable, with retry button) +7. **Status badge**: Idle / Running / Paused / Completed / Failed + +### Task 3.5: Integrity verification post-migration + +After migration completes (before `complete_migration`): +1. Run `verify_integrity()` against the target backend +2. Compare blob count in PG vs target backend +3. Sample-verify N random blobs (download + BLAKE3 hash check) +4. Show verification results in admin UI before allowing finalization + +--- + +## Phase 4 — Enterprise Extras + +### Task 4.1: `CachedBlobBackend` — LRU local disk cache + +**File**: `src/infrastructure/services/cached_blob_backend.rs` (NEW) + +A `BlobStorageBackend` decorator for remote backends (S3, Azure) that caches hot blobs on local SSD: + +```rust +pub struct CachedBlobBackend { + inner: Arc, // S3 backend + cache_dir: PathBuf, // local cache directory + max_cache_bytes: u64, // configurable limit + index: Arc>>, + current_size: Arc, +} + +struct CacheEntry { + size: u64, + last_accessed: Instant, +} +``` + +**Behavior**: +- **Reads**: Check local cache first → cache hit returns local file stream → cache miss downloads from inner backend, writes to cache, returns stream +- **Writes**: `put_blob()` writes to inner backend AND local cache simultaneously +- **Eviction**: LRU eviction when `current_size` exceeds `max_cache_bytes` +- **Startup**: Scan cache directory to rebuild index + +**Configuration** (admin panel): +``` +storage.cache.enabled → "true" | "false" +storage.cache.max_size_bytes → u64 (default: 50 GB) +storage.cache.path → PathBuf (default: "{storage_root}/.cache") +``` + +**Env vars**: +``` +OXICLOUD_STORAGE_CACHE_ENABLED=true +OXICLOUD_STORAGE_CACHE_MAX_SIZE=53687091200 # 50 GB +OXICLOUD_STORAGE_CACHE_PATH=/fast-ssd/oxicloud-cache +``` + +### Task 4.2: Client-side encryption (AES-256-GCM) + +**File**: `src/infrastructure/services/encrypted_blob_backend.rs` (NEW) + +Another `BlobStorageBackend` decorator that encrypts blobs before sending to the inner backend: + +```rust +pub struct EncryptedBlobBackend { + inner: Arc, + encryption_key: [u8; 32], // AES-256 key +} +``` + +**Dependency**: Add `aes-gcm = "0.10"` to Cargo.toml. + +**Behavior**: +- `put_blob()`: Read source → encrypt with AES-256-GCM (random 96-bit nonce prepended) → write encrypted to temp → `inner.put_blob(hash, encrypted_temp)` +- `get_blob_stream()`: `inner.get_blob_stream()` → decrypt stream → return plaintext stream +- **CRITICAL**: BLAKE3 hash is computed on the PLAINTEXT (before encryption), so dedup still works across encrypted backends +- **Nonce storage**: Prepend 12-byte nonce to each encrypted blob (total overhead: 28 bytes per blob — 12 nonce + 16 GCM tag) + +**Configuration** (admin panel): +``` +storage.encryption.enabled → "true" | "false" +storage.encryption.key → base64-encoded 32-byte key (is_secret: true) +``` + +**Key generation**: Provide an admin API endpoint `POST /api/admin/settings/storage/generate-key` that generates a cryptographically secure key and returns it once (user must save it). + +**WARNING in admin UI**: "If you lose the encryption key, all data in the storage backend is IRRECOVERABLY LOST. Back up this key securely." + +### Task 4.3: Azure Blob Storage backend + +**File**: `src/infrastructure/services/azure_blob_backend.rs` (NEW) + +**Dependency**: `azure_storage_blobs = "0.21"`, `azure_storage = "0.21"` + +Same `BlobStorageBackend` trait implementation targeting Azure Blob Storage: +- Container = bucket equivalent +- Blob key scheme: `{prefix}/{hash}.blob` (same as S3/local) +- Authentication: Account Name + Account Key OR SAS token + +**Configuration**: +``` +OXICLOUD_AZURE_ACCOUNT_NAME +OXICLOUD_AZURE_ACCOUNT_KEY +OXICLOUD_AZURE_CONTAINER +OXICLOUD_AZURE_SAS_TOKEN # alternative auth +``` + +### Task 4.4: Bandwidth throttling & retry policies + +Apply to all remote backends (S3, Azure): + +**Throttling**: +- Configurable upload/download bandwidth limit per-backend +- Implemented via `tokio::time::sleep` between chunks +- Admin configurable: `storage.s3.max_upload_bandwidth`, `storage.s3.max_download_bandwidth` + +**Retry policy** (exponential backoff): +```rust +pub struct RetryPolicy { + pub max_retries: u32, // default: 3 + pub initial_backoff_ms: u64, // default: 100 + pub max_backoff_ms: u64, // default: 10_000 + pub backoff_multiplier: f64, // default: 2.0 +} +``` + +Wrap remote backend calls with retry logic for transient errors (network timeouts, 500s, 503s). + +--- + +## Architecture Invariants (MUST be preserved) + +1. **BLAKE3 hashing**: Always performed locally on temp files, never delegated to backend +2. **PostgreSQL dedup index**: `storage.blobs` table remains the source of truth for ref_count, metadata +3. **Write-first strategy**: Blob stored in backend BEFORE PostgreSQL upsert +4. **Remove-after-commit**: Blob deleted from backend AFTER PostgreSQL transaction commits +5. **Streaming reads**: All blob reads return `Pin>`, never load full blob into memory +6. **Zero framework deps in domain**: `BlobStorageBackend` trait lives in `application/ports/`, not infrastructure +7. **DI via AppState**: All backends are `Arc`-wrapped and assembled in `common/di.rs` +8. **Env var precedence**: `OXICLOUD_*` env vars always override DB-stored admin settings +9. **Admin guard**: All storage admin endpoints require JWT with role `"admin"` +10. **Existing tests**: All ~208 tests must pass after Phase 1 refactor + +## Backend Decorator Composition + +The backends compose as decorators. In `di.rs`, the assembly looks like: + +```rust +// Phase 1: Base backend +let base_backend: Arc = match config { + Local => Arc::new(LocalBlobBackend::new(...)), + S3 => Arc::new(S3BlobBackend::new(...).await?), + Azure => Arc::new(AzureBlobBackend::new(...).await?), +}; + +// Phase 4: Optional encryption layer +let backend = if encryption_enabled { + Arc::new(EncryptedBlobBackend::new(base_backend, key)) +} else { + base_backend +}; + +// Phase 4: Optional cache layer (only for remote backends) +let backend = if cache_enabled && !matches!(config, Local) { + Arc::new(CachedBlobBackend::new(backend, cache_config)) +} else { + backend +}; + +// Phase 3: Optional migration wrapper +let backend = if migration_in_progress { + Arc::new(MigrationBlobBackend::new(old_backend, backend, state)) +} else { + backend +}; + +// Finally: DedupService uses the composed backend +let dedup_service = Arc::new(DedupService::new(backend, pool, maintenance_pool)); +``` + +This decorator pattern means each feature (encryption, caching, migration) is: +- Independently testable +- Independently toggleable +- Zero overhead when disabled +- Composable in any order diff --git a/src/application/dtos/settings_dto.rs b/src/application/dtos/settings_dto.rs index 4aaf4a4e..802ff064 100644 --- a/src/application/dtos/settings_dto.rs +++ b/src/application/dtos/settings_dto.rs @@ -128,3 +128,97 @@ pub struct DashboardStatsDto { pub users_over_quota: i64, pub registration_enabled: bool, } + +// ============================================================================ +// Storage Settings DTOs (Admin Panel) +// ============================================================================ + +/// Current storage settings returned to admin UI (secrets masked) +#[derive(Debug, Serialize, Deserialize)] +pub struct StorageSettingsDto { + /// Active backend type: "local" or "s3" + pub backend: String, + pub s3_endpoint_url: Option, + pub s3_bucket: Option, + pub s3_region: Option, + /// True if an access key is configured (never reveals the actual value) + pub s3_access_key_set: bool, + /// True if a secret key is configured (never reveals the actual value) + pub s3_secret_key_set: bool, + pub s3_force_path_style: bool, + /// Field names overridden by environment variables (read-only in UI) + pub env_overrides: Vec, + // ── Current stats ── + pub current_backend: String, + pub total_blobs: u64, + pub total_bytes_stored: u64, + pub dedup_ratio: f64, +} + +/// Request body for saving storage settings from the admin panel +#[derive(Debug, Serialize, Deserialize)] +pub struct SaveStorageSettingsDto { + pub backend: String, + pub s3_endpoint_url: Option, + pub s3_bucket: Option, + pub s3_region: Option, + /// Only update if provided and non-empty (None = keep existing) + pub s3_access_key: Option, + /// Only update if provided and non-empty (None = keep existing) + pub s3_secret_key: Option, + pub s3_force_path_style: Option, +} + +/// Request body for testing a storage connection +#[derive(Debug, Serialize, Deserialize)] +pub struct TestStorageConnectionDto { + pub backend: String, + pub s3_endpoint_url: Option, + pub s3_bucket: Option, + pub s3_region: Option, + pub s3_access_key: Option, + pub s3_secret_key: Option, + pub s3_force_path_style: Option, +} + +/// Result of a storage connection test +#[derive(Debug, Serialize, Deserialize)] +pub struct StorageTestResultDto { + pub connected: bool, + pub message: String, + pub backend_type: String, + pub available_bytes: Option, +} + +// ============================================================================ +// Migration DTOs (Admin Panel — Storage Migration) +// ============================================================================ + +/// Migration progress returned by `GET /api/admin/storage/migration`. +/// Re-exports the `MigrationState` shape for the admin UI. +#[derive(Debug, Serialize, Deserialize)] +pub struct MigrationStateDto { + pub status: String, + pub total_blobs: u64, + pub migrated_blobs: u64, + pub migrated_bytes: u64, + pub failed_blobs: Vec, + pub started_at: Option, + pub completed_at: Option, + /// Estimated throughput in bytes/sec (for UI ETA calculation). + pub throughput_bytes_per_sec: Option, +} + +/// Request body for `POST /api/admin/storage/migration/start`. +#[derive(Debug, Serialize, Deserialize)] +pub struct StartMigrationDto { + /// How many blobs to copy in parallel (default: 4). + pub concurrency: Option, +} + +/// Request body (empty) for `POST /api/admin/storage/migration/verify`. +#[derive(Debug, Serialize, Deserialize)] +pub struct VerifyMigrationDto { + /// Number of random blobs to sample-check (default: 100). + pub sample_size: Option, +} diff --git a/src/application/ports/blob_storage_ports.rs b/src/application/ports/blob_storage_ports.rs new file mode 100644 index 00000000..a5decd58 --- /dev/null +++ b/src/application/ports/blob_storage_ports.rs @@ -0,0 +1,88 @@ +//! Blob Storage Backend Port — abstracts raw byte I/O for content-addressable storage. +//! +//! This trait decouples `DedupService` from any specific storage medium. +//! Implementations include: +//! - `LocalBlobBackend` — local filesystem (default) +//! - `S3BlobBackend` — any S3-compatible service (AWS, Backblaze B2, MinIO, R2…) +//! +//! `DedupService` owns an `Arc` and delegates all +//! byte-level I/O through this trait, keeping BLAKE3 hashing, ref-counting +//! and PostgreSQL index logic in `DedupService` itself. + +use bytes::Bytes; +use futures::Stream; +use serde::Serialize; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; + +use crate::domain::errors::DomainError; + +/// Boxed future alias used by [`BlobStorageBackend`] to keep the trait dyn-compatible. +type BoxFut<'a, T> = Pin + Send + 'a>>; + +/// Pinned boxed byte stream — the return type for blob reads. +pub type BlobStream = Pin> + Send>>; + +/// Health-check result returned by [`BlobStorageBackend::health_check`]. +#[derive(Debug, Clone, Serialize)] +pub struct StorageHealthStatus { + /// Whether the backend is reachable and functional. + pub connected: bool, + /// Human-readable backend identifier (e.g. `"local"`, `"s3"`). + pub backend_type: String, + /// Descriptive status message. + pub message: String, + /// Available space in bytes, if the backend can report it. + pub available_bytes: Option, +} + +/// Minimal trait for blob byte I/O — decoupled from dedup logic. +/// +/// Every method operates on a *hash key* that uniquely identifies a blob. +/// The backend is responsible for mapping the hash to its own addressing +/// scheme (filesystem path, S3 key, etc.). +/// +/// Returns boxed futures so the trait is dyn-compatible (`Arc`). +pub trait BlobStorageBackend: Send + Sync + 'static { + /// Perform any one-time setup (create directories, verify bucket, etc.). + fn initialize(&self) -> BoxFut<'_, Result<(), DomainError>>; + + /// Store a blob from a local temporary file. + /// + /// Must be **idempotent**: if the blob already exists the call succeeds + /// without overwriting. Returns the number of bytes stored. + fn put_blob(&self, hash: &str, source_path: &Path) -> BoxFut<'_, Result>; + + /// Stream the full blob content in chunks. + fn get_blob_stream(&self, hash: &str) -> BoxFut<'_, Result>; + + /// Stream a byte range of the blob (for HTTP Range requests / video seek). + fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> BoxFut<'_, Result>; + + /// Delete a blob by hash. Must be **idempotent** (no error if already gone). + fn delete_blob(&self, hash: &str) -> BoxFut<'_, Result<(), DomainError>>; + + /// Check if a blob exists in the backend. + fn blob_exists(&self, hash: &str) -> BoxFut<'_, Result>; + + /// Get blob size in bytes without downloading content. + fn blob_size(&self, hash: &str) -> BoxFut<'_, Result>; + + /// Verify connectivity and permissions (used by the admin "Test Connection" button). + fn health_check(&self) -> BoxFut<'_, Result>; + + /// Return the backend type name for display (e.g. `"local"`, `"s3"`). + fn backend_type(&self) -> &'static str; + + /// Return the local filesystem path for a blob, if available. + /// + /// Only meaningful for local-filesystem backends. Remote backends + /// return `None`; callers that need a local file must stream + spool. + fn local_blob_path(&self, hash: &str) -> Option; +} diff --git a/src/application/ports/mod.rs b/src/application/ports/mod.rs index a2fc2bf7..22302921 100644 --- a/src/application/ports/mod.rs +++ b/src/application/ports/mod.rs @@ -1,4 +1,5 @@ pub mod auth_ports; +pub mod blob_storage_ports; pub mod cache_ports; pub mod calendar_ports; pub mod carddav_ports; diff --git a/src/application/services/mod.rs b/src/application/services/mod.rs index 98991b93..27145ef6 100644 --- a/src/application/services/mod.rs +++ b/src/application/services/mod.rs @@ -18,6 +18,7 @@ pub mod nextcloud_login_flow_service; pub mod recent_service; pub mod search_service; pub mod share_service; +pub mod storage_settings_service; pub mod storage_usage_service; pub mod trash_service; pub mod wopi_lock_service; diff --git a/src/application/services/storage_settings_service.rs b/src/application/services/storage_settings_service.rs new file mode 100644 index 00000000..1d9fbf94 --- /dev/null +++ b/src/application/services/storage_settings_service.rs @@ -0,0 +1,337 @@ +use std::collections::HashMap; +use std::sync::Arc; +use uuid::Uuid; + +use crate::application::dtos::settings_dto::{ + SaveStorageSettingsDto, StorageSettingsDto, StorageTestResultDto, TestStorageConnectionDto, +}; +use crate::application::ports::blob_storage_ports::BlobStorageBackend; +use crate::common::config::{S3StorageConfig, StorageConfig}; +use crate::common::errors::{DomainError, ErrorKind}; +use crate::domain::repositories::settings_repository::SettingsRepository; +use crate::infrastructure::repositories::pg::SettingsPgRepository; +use crate::infrastructure::services::dedup_service::DedupService; +use crate::infrastructure::services::s3_blob_backend::S3BlobBackend; + +/// Storage settings service — manages storage backend configuration via the admin panel. +/// +/// Configuration priority: **env vars > DB settings > defaults**. +pub struct StorageSettingsService { + settings_repo: Arc, + env_storage_config: StorageConfig, + dedup_service: Arc, +} + +impl StorageSettingsService { + pub fn new( + settings_repo: Arc, + env_storage_config: StorageConfig, + dedup_service: Arc, + ) -> Self { + Self { + settings_repo, + env_storage_config, + dedup_service, + } + } + + /// Detect which storage fields are overridden by environment variables. + fn get_env_overrides(&self) -> Vec { + let mut out = Vec::new(); + let vars = [ + ("OXICLOUD_STORAGE_BACKEND", "backend"), + ("OXICLOUD_S3_ENDPOINT_URL", "s3_endpoint_url"), + ("OXICLOUD_S3_BUCKET", "s3_bucket"), + ("OXICLOUD_S3_REGION", "s3_region"), + ("OXICLOUD_S3_ACCESS_KEY", "s3_access_key"), + ("OXICLOUD_S3_SECRET_KEY", "s3_secret_key"), + ("OXICLOUD_S3_FORCE_PATH_STYLE", "s3_force_path_style"), + ]; + for (env_key, field_name) in &vars { + if std::env::var(env_key).is_ok() { + out.push(field_name.to_string()); + } + } + out + } + + /// Apply environment variable overrides on top of a config. + fn apply_env_overrides(&self, config: &mut StorageConfig) { + let e = &self.env_storage_config; + if std::env::var("OXICLOUD_STORAGE_BACKEND").is_ok() { + config.backend = e.backend.clone(); + } + // S3 env overrides — only apply if S3 config exists in env + if let Some(env_s3) = &e.s3 { + let s3 = config.s3.get_or_insert_with(|| S3StorageConfig { + endpoint_url: None, + bucket: String::new(), + region: "us-east-1".to_string(), + access_key: String::new(), + secret_key: String::new(), + force_path_style: false, + }); + if std::env::var("OXICLOUD_S3_ENDPOINT_URL").is_ok() { + s3.endpoint_url = env_s3.endpoint_url.clone(); + } + if std::env::var("OXICLOUD_S3_BUCKET").is_ok() { + s3.bucket = env_s3.bucket.clone(); + } + if std::env::var("OXICLOUD_S3_REGION").is_ok() { + s3.region = env_s3.region.clone(); + } + if std::env::var("OXICLOUD_S3_ACCESS_KEY").is_ok() { + s3.access_key = env_s3.access_key.clone(); + } + if std::env::var("OXICLOUD_S3_SECRET_KEY").is_ok() { + s3.secret_key = env_s3.secret_key.clone(); + } + if std::env::var("OXICLOUD_S3_FORCE_PATH_STYLE").is_ok() { + s3.force_path_style = env_s3.force_path_style; + } + } + } + + /// Load effective storage config: DB settings + env var overrides + defaults. + pub async fn load_effective_storage_config(&self) -> Result { + let db: HashMap = self.settings_repo.get_by_category("storage").await?; + let d = StorageConfig::default(); + + let backend = db + .get("storage.backend") + .map(|v| match v.as_str() { + "s3" => crate::common::config::StorageBackendType::S3, + "azure" => crate::common::config::StorageBackendType::Azure, + _ => crate::common::config::StorageBackendType::Local, + }) + .unwrap_or(d.backend); + + let s3 = { + let bucket = db.get("storage.s3.bucket").cloned().unwrap_or_default(); + if bucket.is_empty() { + None + } else { + Some(S3StorageConfig { + endpoint_url: db + .get("storage.s3.endpoint_url") + .cloned() + .filter(|s| !s.is_empty()), + bucket, + region: db + .get("storage.s3.region") + .cloned() + .unwrap_or_else(|| "us-east-1".to_string()), + access_key: db.get("storage.s3.access_key").cloned().unwrap_or_default(), + secret_key: db.get("storage.s3.secret_key").cloned().unwrap_or_default(), + force_path_style: db + .get("storage.s3.force_path_style") + .and_then(|v| v.parse().ok()) + .unwrap_or(false), + }) + } + }; + + let mut config = StorageConfig { + backend, + s3, + ..self.env_storage_config.clone() + }; + + self.apply_env_overrides(&mut config); + Ok(config) + } + + /// Get storage settings for display in admin UI (secrets masked). + pub async fn get_storage_settings(&self) -> Result { + let db: HashMap = self.settings_repo.get_by_category("storage").await?; + + let has_access_key = db + .get("storage.s3.access_key") + .map(|s| !s.is_empty()) + .unwrap_or(false) + || std::env::var("OXICLOUD_S3_ACCESS_KEY") + .map(|s| !s.is_empty()) + .unwrap_or(false); + + let has_secret_key = db + .get("storage.s3.secret_key") + .map(|s| !s.is_empty()) + .unwrap_or(false) + || std::env::var("OXICLOUD_S3_SECRET_KEY") + .map(|s| !s.is_empty()) + .unwrap_or(false); + + let effective = self.load_effective_storage_config().await?; + let stats = self.dedup_service.get_stats().await; + let current_backend = self.dedup_service.backend().backend_type().to_string(); + + let backend_str = match effective.backend { + crate::common::config::StorageBackendType::Local => "local", + crate::common::config::StorageBackendType::S3 => "s3", + crate::common::config::StorageBackendType::Azure => "azure", + }; + + Ok(StorageSettingsDto { + backend: backend_str.to_string(), + s3_endpoint_url: effective.s3.as_ref().and_then(|s| s.endpoint_url.clone()), + s3_bucket: effective.s3.as_ref().map(|s| s.bucket.clone()), + s3_region: effective.s3.as_ref().map(|s| s.region.clone()), + s3_access_key_set: has_access_key, + s3_secret_key_set: has_secret_key, + s3_force_path_style: effective.s3.as_ref().is_some_and(|s| s.force_path_style), + env_overrides: self.get_env_overrides(), + current_backend, + total_blobs: stats.total_blobs, + total_bytes_stored: stats.total_bytes_stored, + dedup_ratio: stats.dedup_ratio, + }) + } + + /// Save storage settings to DB. + pub async fn save_storage_settings( + &self, + dto: SaveStorageSettingsDto, + updated_by: Uuid, + ) -> Result<(), DomainError> { + let cat = "storage"; + let by = Some(updated_by); + + self.settings_repo + .set("storage.backend", &dto.backend, cat, false, by) + .await?; + + if let Some(ref v) = dto.s3_endpoint_url { + self.settings_repo + .set("storage.s3.endpoint_url", v, cat, false, by) + .await?; + } + if let Some(ref v) = dto.s3_bucket { + self.settings_repo + .set("storage.s3.bucket", v, cat, false, by) + .await?; + } + if let Some(ref v) = dto.s3_region { + self.settings_repo + .set("storage.s3.region", v, cat, false, by) + .await?; + } + if let Some(ref v) = dto.s3_access_key + && !v.is_empty() + { + self.settings_repo + .set("storage.s3.access_key", v, cat, true, by) + .await?; + } + if let Some(ref v) = dto.s3_secret_key + && !v.is_empty() + { + self.settings_repo + .set("storage.s3.secret_key", v, cat, true, by) + .await?; + } + if let Some(v) = dto.s3_force_path_style { + self.settings_repo + .set( + "storage.s3.force_path_style", + &v.to_string(), + cat, + false, + by, + ) + .await?; + } + + tracing::info!("Storage settings saved by admin (backend={})", dto.backend); + Ok(()) + } + + /// Test a storage connection by building a temporary backend and calling health_check(). + pub async fn test_storage_connection( + &self, + dto: TestStorageConnectionDto, + ) -> Result { + match dto.backend.as_str() { + "local" => { + // Test local backend health via the current dedup service backend + let status = self.dedup_service.backend().health_check().await?; + Ok(StorageTestResultDto { + connected: status.connected, + message: status.message, + backend_type: "local".to_string(), + available_bytes: status.available_bytes, + }) + } + "s3" => { + let bucket = dto.s3_bucket.as_deref().unwrap_or_default(); + if bucket.is_empty() { + return Ok(StorageTestResultDto { + connected: false, + message: "S3 bucket name is required".to_string(), + backend_type: "s3".to_string(), + available_bytes: None, + }); + } + + // Build a temporary S3 backend from the DTO values, + // falling back to existing DB/env config for missing fields. + let effective = self.load_effective_storage_config().await.ok(); + let existing_s3 = effective.as_ref().and_then(|c| c.s3.as_ref()); + + let config = S3StorageConfig { + endpoint_url: dto + .s3_endpoint_url + .clone() + .or_else(|| existing_s3.and_then(|s| s.endpoint_url.clone())), + bucket: bucket.to_string(), + region: dto.s3_region.clone().unwrap_or_else(|| { + existing_s3 + .map(|s| s.region.clone()) + .unwrap_or_else(|| "us-east-1".to_string()) + }), + access_key: dto + .s3_access_key + .clone() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| { + existing_s3 + .map(|s| s.access_key.clone()) + .unwrap_or_default() + }), + secret_key: dto + .s3_secret_key + .clone() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| { + existing_s3 + .map(|s| s.secret_key.clone()) + .unwrap_or_default() + }), + force_path_style: dto + .s3_force_path_style + .unwrap_or_else(|| existing_s3.is_some_and(|s| s.force_path_style)), + }; + + let backend = S3BlobBackend::new(&config); + match backend.health_check().await { + Ok(status) => Ok(StorageTestResultDto { + connected: status.connected, + message: status.message, + backend_type: "s3".to_string(), + available_bytes: status.available_bytes, + }), + Err(e) => Ok(StorageTestResultDto { + connected: false, + message: format!("Connection failed: {}", e), + backend_type: "s3".to_string(), + available_bytes: None, + }), + } + } + other => Err(DomainError::new( + ErrorKind::InvalidInput, + "Storage", + format!("Unknown backend type: {}", other), + )), + } + } +} diff --git a/src/common/config.rs b/src/common/config.rs index 7330b8b3..dc2a43d7 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -211,6 +211,127 @@ pub struct StorageConfig { /// Maximum upload file size in bytes (default: 10 GB). /// Applied as a hard limit to WebDAV PUT and streaming uploads. pub max_upload_size: usize, + /// Which blob storage backend to use (`local`, `s3`, or `azure`). + pub backend: StorageBackendType, + /// S3-compatible backend configuration (used when `backend == S3`). + pub s3: Option, + /// Azure Blob Storage configuration (used when `backend == Azure`). + pub azure: Option, + /// Local disk cache for remote backends. + pub cache: BlobCacheConfig, + /// Client-side encryption. + pub encryption: EncryptionConfig, + /// Retry policy for remote backends. + pub retry: RetryConfig, +} + +/// Which blob storage backend to use. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub enum StorageBackendType { + /// Local filesystem (default). + #[default] + Local, + /// Any S3-compatible object store (AWS, Backblaze B2, R2, MinIO, …). + S3, + /// Azure Blob Storage. + Azure, +} + +/// Configuration for an S3-compatible blob storage backend. +#[derive(Debug, Clone)] +pub struct S3StorageConfig { + /// Custom endpoint URL (required for non-AWS providers). + pub endpoint_url: Option, + /// S3 bucket name. + pub bucket: String, + /// AWS region (default: `us-east-1`). + pub region: String, + /// Access key ID. + pub access_key: String, + /// Secret access key. + pub secret_key: String, + /// Force path-style access (required for MinIO, R2, some providers). + pub force_path_style: bool, +} + +/// Configuration for Azure Blob Storage. +#[derive(Debug, Clone)] +pub struct AzureStorageConfig { + /// Azure storage account name. + pub account_name: String, + /// Azure storage account key. + pub account_key: String, + /// Container name. + pub container: String, + /// Optional SAS token (alternative to account key). + pub sas_token: Option, +} + +/// LRU local disk cache configuration for remote blob backends. +#[derive(Debug, Clone)] +pub struct BlobCacheConfig { + /// Enable the LRU disk cache (only useful for remote backends). + pub enabled: bool, + /// Maximum cache size in bytes (default: 50 GB). + pub max_size_bytes: u64, + /// Cache directory path (default: `{root_dir}/.blob-cache`). + pub cache_path: Option, +} + +impl Default for BlobCacheConfig { + fn default() -> Self { + Self { + enabled: false, + max_size_bytes: 50 * 1024 * 1024 * 1024, // 50 GB + cache_path: None, + } + } +} + +/// Client-side encryption configuration. +#[derive(Debug, Clone)] +pub struct EncryptionConfig { + /// Enable AES-256-GCM encryption for blobs at rest. + pub enabled: bool, + /// Base64-encoded 32-byte encryption key. + pub key_base64: Option, +} + +impl Default for EncryptionConfig { + #[allow(clippy::derivable_impls)] + fn default() -> Self { + Self { + enabled: false, + key_base64: None, + } + } +} + +/// Retry policy configuration for remote backends. +#[derive(Debug, Clone)] +pub struct RetryConfig { + /// Enable retry with exponential backoff. + pub enabled: bool, + /// Maximum number of retry attempts. + pub max_retries: u32, + /// Initial backoff in milliseconds. + pub initial_backoff_ms: u64, + /// Maximum backoff in milliseconds. + pub max_backoff_ms: u64, + /// Backoff multiplier. + pub backoff_multiplier: f64, +} + +impl Default for RetryConfig { + fn default() -> Self { + Self { + enabled: true, + max_retries: 3, + initial_backoff_ms: 100, + max_backoff_ms: 10_000, + backoff_multiplier: 2.0, + } + } } impl Default for StorageConfig { @@ -227,6 +348,12 @@ impl Default for StorageConfig { parallel_threshold: 100 * 1024 * 1024, // 100 MB trash_retention_days: 30, // 30 days max_upload_size: MAX_UPLOAD_SIZE, + backend: StorageBackendType::Local, + s3: None, + azure: None, + cache: BlobCacheConfig::default(), + encryption: EncryptionConfig::default(), + retry: RetryConfig::default(), } } } @@ -828,6 +955,95 @@ impl AppConfig { config.storage.max_upload_size = val; } + // Storage backend selection + if let Ok(backend) = env::var("OXICLOUD_STORAGE_BACKEND") { + match backend.to_lowercase().as_str() { + "s3" => config.storage.backend = StorageBackendType::S3, + "azure" => config.storage.backend = StorageBackendType::Azure, + _ => config.storage.backend = StorageBackendType::Local, + } + } + + // S3-compatible storage configuration + if config.storage.backend == StorageBackendType::S3 { + let bucket = env::var("OXICLOUD_S3_BUCKET").unwrap_or_default(); + if bucket.is_empty() { + tracing::warn!("OXICLOUD_STORAGE_BACKEND=s3 but OXICLOUD_S3_BUCKET is not set"); + } + config.storage.s3 = Some(S3StorageConfig { + endpoint_url: env::var("OXICLOUD_S3_ENDPOINT_URL").ok(), + bucket, + region: env::var("OXICLOUD_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()), + access_key: env::var("OXICLOUD_S3_ACCESS_KEY").unwrap_or_default(), + secret_key: env::var("OXICLOUD_S3_SECRET_KEY").unwrap_or_default(), + force_path_style: env::var("OXICLOUD_S3_FORCE_PATH_STYLE") + .map(|v| v.parse::().unwrap_or(false)) + .unwrap_or(false), + }); + } + + // Azure Blob Storage configuration + if config.storage.backend == StorageBackendType::Azure { + let container = env::var("OXICLOUD_AZURE_CONTAINER").unwrap_or_default(); + if container.is_empty() { + tracing::warn!( + "OXICLOUD_STORAGE_BACKEND=azure but OXICLOUD_AZURE_CONTAINER is not set" + ); + } + config.storage.azure = Some(AzureStorageConfig { + account_name: env::var("OXICLOUD_AZURE_ACCOUNT_NAME").unwrap_or_default(), + account_key: env::var("OXICLOUD_AZURE_ACCOUNT_KEY").unwrap_or_default(), + container, + sas_token: env::var("OXICLOUD_AZURE_SAS_TOKEN").ok(), + }); + } + + // Blob cache configuration + if let Ok(v) = env::var("OXICLOUD_STORAGE_CACHE_ENABLED") { + config.storage.cache.enabled = v.parse::().unwrap_or(false); + } + if let Ok(v) = env::var("OXICLOUD_STORAGE_CACHE_MAX_SIZE") + && let Ok(bytes) = v.parse::() + { + config.storage.cache.max_size_bytes = bytes; + } + if let Ok(v) = env::var("OXICLOUD_STORAGE_CACHE_PATH") { + config.storage.cache.cache_path = Some(v); + } + + // Encryption configuration + if let Ok(v) = env::var("OXICLOUD_STORAGE_ENCRYPTION_ENABLED") { + config.storage.encryption.enabled = v.parse::().unwrap_or(false); + } + if let Ok(v) = env::var("OXICLOUD_STORAGE_ENCRYPTION_KEY") { + config.storage.encryption.key_base64 = Some(v); + } + + // Retry configuration + if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_ENABLED") { + config.storage.retry.enabled = v.parse::().unwrap_or(true); + } + if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_MAX_RETRIES") + && let Ok(n) = v.parse::() + { + config.storage.retry.max_retries = n; + } + if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_INITIAL_BACKOFF_MS") + && let Ok(n) = v.parse::() + { + config.storage.retry.initial_backoff_ms = n; + } + if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_MAX_BACKOFF_MS") + && let Ok(n) = v.parse::() + { + config.storage.retry.max_backoff_ms = n; + } + if let Ok(v) = env::var("OXICLOUD_STORAGE_RETRY_BACKOFF_MULTIPLIER") + && let Ok(n) = v.parse::() + { + config.storage.retry.backoff_multiplier = n; + } + // OIDC configuration if let Ok(v) = env::var("OXICLOUD_OIDC_ENABLED") { config.oidc.enabled = v.parse::().unwrap_or(false); diff --git a/src/common/di.rs b/src/common/di.rs index 69c70a1e..3334e63d 100644 --- a/src/common/di.rs +++ b/src/common/di.rs @@ -2,10 +2,14 @@ use sqlx::PgPool; use std::path::PathBuf; use std::sync::Arc; +use crate::application::ports::blob_storage_ports::BlobStorageBackend; +use crate::common::config::StorageBackendType; use crate::infrastructure::db::DbPools; use crate::application::services::admin_settings_service::AdminSettingsService; use crate::application::services::auth_application_service::AuthApplicationService; +use crate::application::services::storage_settings_service::StorageSettingsService; +use crate::infrastructure::services::migration_blob_backend::MigrationState; use crate::application::ports::file_ports::FileUseCaseFactory; use crate::application::services::favorites_service::FavoritesService; @@ -153,10 +157,110 @@ impl AppServiceFactory { ); image_transcode_service.initialize().await?; + // Build blob storage backend based on configuration + let base_backend: Arc = match self.config.storage.backend { + StorageBackendType::S3 => { + let s3_config = self + .config + .storage + .s3 + .as_ref() + .expect("S3 config required when OXICLOUD_STORAGE_BACKEND=s3"); + Arc::new( + crate::infrastructure::services::s3_blob_backend::S3BlobBackend::new(s3_config), + ) + } + StorageBackendType::Azure => { + let az_config = self + .config + .storage + .azure + .as_ref() + .expect("Azure config required when OXICLOUD_STORAGE_BACKEND=azure"); + Arc::new( + crate::infrastructure::services::azure_blob_backend::AzureBlobBackend::new( + az_config, + ), + ) + } + StorageBackendType::Local => Arc::new( + crate::infrastructure::services::local_blob_backend::LocalBlobBackend::new( + &self.storage_path, + ), + ), + }; + + // Stack decorators: retry → encryption → cache (inner-to-outer) + let mut blob_backend: Arc = base_backend; + + // Retry decorator (for remote backends) + if self.config.storage.retry.enabled + && self.config.storage.backend != StorageBackendType::Local + { + use crate::infrastructure::services::retry_blob_backend::{ + RetryBlobBackend, RetryPolicy, + }; + let policy = RetryPolicy { + max_retries: self.config.storage.retry.max_retries, + initial_backoff: std::time::Duration::from_millis( + self.config.storage.retry.initial_backoff_ms, + ), + max_backoff: std::time::Duration::from_millis( + self.config.storage.retry.max_backoff_ms, + ), + backoff_multiplier: self.config.storage.retry.backoff_multiplier, + }; + blob_backend = Arc::new(RetryBlobBackend::new(blob_backend, policy)); + tracing::info!("Blob storage retry decorator enabled"); + } + + // Encryption decorator + if self.config.storage.encryption.enabled { + use crate::infrastructure::services::encrypted_blob_backend::EncryptedBlobBackend; + let key_b64 = self + .config + .storage + .encryption + .key_base64 + .as_ref() + .expect("OXICLOUD_STORAGE_ENCRYPTION_KEY required when encryption is enabled"); + let key_bytes = + base64::Engine::decode(&base64::engine::general_purpose::STANDARD, key_b64) + .expect("OXICLOUD_STORAGE_ENCRYPTION_KEY must be valid base64"); + let key: [u8; 32] = key_bytes.try_into().expect( + "OXICLOUD_STORAGE_ENCRYPTION_KEY must be exactly 32 bytes (base64 of 32 bytes)", + ); + blob_backend = Arc::new(EncryptedBlobBackend::new(blob_backend, &key)); + tracing::info!("Blob storage encryption decorator enabled (AES-256-GCM)"); + } + + // Cache decorator (for remote backends only) + if self.config.storage.cache.enabled + && self.config.storage.backend != StorageBackendType::Local + { + use crate::infrastructure::services::cached_blob_backend::{ + BlobCacheConfig as CacheCfg, CachedBlobBackend, + }; + let cache_path = self + .config + .storage + .cache + .cache_path + .as_ref() + .map(std::path::PathBuf::from) + .unwrap_or_else(|| self.storage_path.join(".blob-cache")); + let cfg = CacheCfg { + cache_dir: cache_path, + max_cache_bytes: self.config.storage.cache.max_size_bytes, + }; + blob_backend = Arc::new(CachedBlobBackend::new(blob_backend, &cfg)); + tracing::info!("Blob storage LRU disk cache enabled"); + } + // Deduplication service — PRIMARY blob storage engine (PostgreSQL-backed index) let dedup_service = Arc::new( crate::infrastructure::services::dedup_service::DedupService::new( - &self.storage_path, + blob_backend, db_pool.clone(), maintenance_pool.clone(), ), @@ -631,6 +735,8 @@ impl AppServiceFactory { auth_service: auth_services, nextcloud: nextcloud_services, admin_settings_service: None, + storage_settings_service: None, + migration_state: Arc::new(tokio::sync::RwLock::new(MigrationState::default())), trash_service, share_service, favorites_service, @@ -700,6 +806,15 @@ impl AppServiceFactory { app_state.admin_settings_service = Some(admin_svc.clone()); + // 9b-1b. Wire storage settings service (reuses same settings_repo) + let storage_settings_svc = Arc::new(StorageSettingsService::new( + settings_repo.clone(), + self.config.storage.clone(), + app_state.core.dedup_service.clone(), + )); + app_state.storage_settings_service = Some(storage_settings_svc); + tracing::info!("Storage settings service initialized"); + // 9b-2. Log whether system needs first-time admin setup if !admin_svc.is_system_initialized().await { tracing::warn!("╔══════════════════════════════════════════════════════════╗"); @@ -928,6 +1043,8 @@ pub struct AppState { pub auth_service: Option, pub nextcloud: Option, pub admin_settings_service: Option>, + pub storage_settings_service: Option>, + pub migration_state: Arc>, pub trash_service: Option>, pub share_service: Option>, pub favorites_service: Option>, diff --git a/src/infrastructure/services/azure_blob_backend.rs b/src/infrastructure/services/azure_blob_backend.rs new file mode 100644 index 00000000..daadb8a6 --- /dev/null +++ b/src/infrastructure/services/azure_blob_backend.rs @@ -0,0 +1,302 @@ +//! Azure Blob Storage Backend — stores blobs in an Azure Storage container. +//! +//! Authenticates via Account Name + Account Key (or SAS token). +//! Blob key scheme mirrors local/S3: `{2-char-prefix}/{hash}.blob`. + +use std::path::{Path, PathBuf}; +use std::pin::Pin; + +use azure_storage::StorageCredentials; +use azure_storage_blobs::prelude::*; +use bytes::Bytes; +use futures::StreamExt; +use tokio::fs; + +use crate::application::ports::blob_storage_ports::{ + BlobStorageBackend, BlobStream, StorageHealthStatus, +}; +use crate::common::config::AzureStorageConfig; +use crate::domain::errors::{DomainError, ErrorKind}; + +/// Azure Blob Storage backend. +pub struct AzureBlobBackend { + container_client: ContainerClient, + container_name: String, +} + +impl AzureBlobBackend { + /// Build a new Azure backend from configuration. + pub fn new(config: &AzureStorageConfig) -> Self { + let credentials = if let Some(ref sas) = config.sas_token { + StorageCredentials::sas_token(sas).expect("Invalid SAS token") + } else { + StorageCredentials::access_key(&config.account_name, config.account_key.clone()) + }; + + let container_client = ClientBuilder::new(&config.account_name, credentials) + .container_client(&config.container); + + Self { + container_client, + container_name: config.container.clone(), + } + } + + /// Compute the blob name for a given hash. + fn blob_name(hash: &str) -> String { + let prefix = &hash[0..2]; + format!("{prefix}/{hash}.blob") + } + + /// Get a `BlobClient` for a given hash. + fn blob_client(&self, hash: &str) -> BlobClient { + self.container_client.blob_client(Self::blob_name(hash)) + } +} + +impl BlobStorageBackend for AzureBlobBackend { + fn initialize( + &self, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + // Verify container exists by getting its properties + self.container_client.get_properties().await.map_err(|e| { + DomainError::internal_error( + "Azure", + format!("Cannot access container '{}': {}", self.container_name, e), + ) + })?; + + tracing::info!( + "Azure blob backend initialized: container={}", + self.container_name + ); + Ok(()) + }) + } + + fn put_blob( + &self, + hash: &str, + source_path: &Path, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + let source_path = source_path.to_owned(); + Box::pin(async move { + let client = self.blob_client(&hash); + + // Check if blob already exists (idempotent) + if client.get_properties().await.is_ok() { + let file_size = fs::metadata(&source_path) + .await + .map_err(|e| { + DomainError::internal_error( + "Azure", + format!("Failed to stat source file: {e}"), + ) + })? + .len(); + let _ = fs::remove_file(&source_path).await; + return Ok(file_size); + } + + // Read file and upload as block blob + let data = fs::read(&source_path).await.map_err(|e| { + DomainError::internal_error("Azure", format!("Failed to read source: {e}")) + })?; + let file_size = data.len() as u64; + + client.put_block_blob(data).await.map_err(|e| { + DomainError::internal_error("Azure", format!("Failed to upload blob {hash}: {e}")) + })?; + + let _ = fs::remove_file(&source_path).await; + Ok(file_size) + }) + } + + fn get_blob_stream( + &self, + hash: &str, + ) -> Pin> + Send + '_>> + { + let hash = hash.to_owned(); + Box::pin(async move { + let client = self.blob_client(&hash); + + let mut result_data: Vec = Vec::new(); + let mut stream = client.get().into_stream(); + + while let Some(response) = stream.next().await { + let response = response.map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "Azure", + format!("Failed to get blob {hash}: {e}"), + ) + })?; + let mut body = response.data; + while let Some(chunk) = body.next().await { + let chunk = chunk.map_err(|e| { + DomainError::internal_error("Azure", format!("Stream read error: {e}")) + })?; + result_data.extend_from_slice(&chunk); + } + } + + let stream: BlobStream = Box::pin(futures::stream::once(async move { + Ok(Bytes::from(result_data)) + })); + Ok(stream) + }) + } + + fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> Pin> + Send + '_>> + { + let hash = hash.to_owned(); + Box::pin(async move { + let client = self.blob_client(&hash); + + let range = match end { + Some(e) => azure_core::request_options::Range::new(start, e), + None => azure_core::request_options::Range::new(start, u64::MAX), + }; + + let mut result_data: Vec = Vec::new(); + let mut stream = client.get().range(range).into_stream(); + + while let Some(response) = stream.next().await { + let response = response.map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "Azure", + format!("Failed to get blob range {hash}: {e}"), + ) + })?; + let mut body = response.data; + while let Some(chunk) = body.next().await { + let chunk = chunk.map_err(|e| { + DomainError::internal_error( + "Azure", + format!("Stream range read error: {e}"), + ) + })?; + result_data.extend_from_slice(&chunk); + } + } + + let stream: BlobStream = Box::pin(futures::stream::once(async move { + Ok(Bytes::from(result_data)) + })); + Ok(stream) + }) + } + + fn delete_blob( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let client = self.blob_client(&hash); + + // Azure delete is not fully idempotent — 404 is expected for missing blobs + match client.delete().await { + Ok(_) => Ok(()), + Err(e) => { + // If 404, treat as success (idempotent) + let status = e.as_http_error().map(|h| h.status()); + if status == Some(azure_core::StatusCode::NotFound) { + Ok(()) + } else { + Err(DomainError::internal_error( + "Azure", + format!("Failed to delete blob {hash}: {e}"), + )) + } + } + } + }) + } + + fn blob_exists( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let client = self.blob_client(&hash); + match client.get_properties().await { + Ok(_) => Ok(true), + Err(e) => { + let status = e.as_http_error().map(|h| h.status()); + if status == Some(azure_core::StatusCode::NotFound) { + Ok(false) + } else { + Err(DomainError::internal_error( + "Azure", + format!("Failed to check blob {hash}: {e}"), + )) + } + } + } + }) + } + + fn blob_size( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let client = self.blob_client(&hash); + let props = client.get_properties().await.map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "Azure", + format!("Failed to stat blob {hash}: {e}"), + ) + })?; + Ok(props.blob.properties.content_length) + }) + } + + fn health_check( + &self, + ) -> Pin< + Box> + Send + '_>, + > { + Box::pin(async move { + match self.container_client.get_properties().await { + Ok(_) => Ok(StorageHealthStatus { + connected: true, + backend_type: "azure".to_string(), + message: format!("Azure container '{}' is accessible", self.container_name), + available_bytes: None, + }), + Err(e) => Ok(StorageHealthStatus { + connected: false, + backend_type: "azure".to_string(), + message: format!( + "Azure container '{}' is not accessible: {}", + self.container_name, e + ), + available_bytes: None, + }), + } + }) + } + + fn backend_type(&self) -> &'static str { + "azure" + } + + fn local_blob_path(&self, _hash: &str) -> Option { + None + } +} diff --git a/src/infrastructure/services/cached_blob_backend.rs b/src/infrastructure/services/cached_blob_backend.rs new file mode 100644 index 00000000..ad3a15ab --- /dev/null +++ b/src/infrastructure/services/cached_blob_backend.rs @@ -0,0 +1,472 @@ +//! `CachedBlobBackend` — LRU local-disk cache decorator for remote blob backends. +//! +//! Wraps any `BlobStorageBackend` (typically S3 or Azure) and transparently +//! caches hot blobs on a local SSD. Reads check the cache first; cache misses +//! are fetched from the inner backend and written to the cache. Writes go to +//! the inner backend AND the local cache simultaneously. +//! +//! Eviction is LRU based on a configurable maximum disk budget. + +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use lru::LruCache; +use std::num::NonZeroUsize; +use tokio::fs; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use tokio::sync::Mutex; +use tokio_util::io::ReaderStream; + +use crate::application::ports::blob_storage_ports::{ + BlobStorageBackend, BlobStream, StorageHealthStatus, +}; +use crate::domain::errors::DomainError; + +/// Chunk size for streaming cached file reads (256 KB). +const STREAM_CHUNK_SIZE: usize = 256 * 1024; + +// ── Configuration ────────────────────────────────────────────────── + +/// Configuration for the LRU disk cache. +#[derive(Debug, Clone)] +pub struct BlobCacheConfig { + /// Directory where cached blobs are stored. + pub cache_dir: PathBuf, + /// Maximum total cache size in bytes. + pub max_cache_bytes: u64, +} + +// ── Cache entry ──────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +struct CacheEntry { + size: u64, +} + +// ── CachedBlobBackend ────────────────────────────────────────────── + +/// A `BlobStorageBackend` decorator that adds an LRU disk cache in front of +/// a remote backend. +pub struct CachedBlobBackend { + inner: Arc, + cache_dir: PathBuf, + max_cache_bytes: u64, + index: Arc>>, + current_size: Arc, +} + +impl CachedBlobBackend { + /// Create a new cached backend wrapping `inner`. + pub fn new(inner: Arc, config: &BlobCacheConfig) -> Self { + Self { + inner, + cache_dir: config.cache_dir.clone(), + max_cache_bytes: config.max_cache_bytes, + // Capacity is essentially unbounded — eviction is by byte budget, not count. + index: Arc::new(Mutex::new(LruCache::new( + NonZeroUsize::new(1_000_000).unwrap(), + ))), + current_size: Arc::new(AtomicU64::new(0)), + } + } + + /// Path where a blob is cached locally. + fn cached_path(&self, hash: &str) -> PathBuf { + let prefix = &hash[..2.min(hash.len())]; + self.cache_dir.join(prefix).join(format!("{hash}.blob")) + } +} + +impl BlobStorageBackend for CachedBlobBackend { + fn initialize( + &self, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let cache_dir = self.cache_dir.clone(); + let index = self.index.clone(); + let current_size = self.current_size.clone(); + Box::pin(async move { + inner.initialize().await?; + + // Create cache dir structure (256 prefix dirs) + fs::create_dir_all(&cache_dir).await.map_err(|e| { + DomainError::internal_error("BlobCache", format!("mkdir cache_dir: {e}")) + })?; + + // Scan existing cache to rebuild index + let mut total_bytes = 0u64; + let mut idx = index.lock().await; + if let Ok(mut read_dir) = fs::read_dir(&cache_dir).await { + while let Ok(Some(prefix_entry)) = read_dir.next_entry().await { + if !prefix_entry.path().is_dir() { + continue; + } + if let Ok(mut sub_dir) = fs::read_dir(prefix_entry.path()).await { + while let Ok(Some(entry)) = sub_dir.next_entry().await { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) == Some("blob") + && let Some(stem) = path.file_stem().and_then(|s| s.to_str()) + { + let size = fs::metadata(&path).await.map(|m| m.len()).unwrap_or(0); + idx.put(stem.to_string(), CacheEntry { size }); + total_bytes += size; + } + } + } + } + } + drop(idx); + current_size.store(total_bytes, Ordering::Relaxed); + tracing::info!( + "Blob cache initialized: {} bytes in cache at {}", + total_bytes, + cache_dir.display() + ); + Ok(()) + }) + } + + fn put_blob( + &self, + hash: &str, + source_path: &Path, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let hash = hash.to_string(); + let source = source_path.to_path_buf(); + let self_ref = CachedRef { + cache_dir: self.cache_dir.clone(), + max_cache_bytes: self.max_cache_bytes, + index: self.index.clone(), + current_size: self.current_size.clone(), + }; + Box::pin(async move { + // Write to inner backend + let bytes = inner.put_blob(&hash, &source).await?; + // Also cache locally (best-effort) + let _ = self_ref.insert_into_cache_static(&hash, &source).await; + Ok(bytes) + }) + } + + fn get_blob_stream( + &self, + hash: &str, + ) -> Pin> + Send + '_>> + { + let hash = hash.to_string(); + let cached = self.cached_path(&hash); + let index = self.index.clone(); + let inner = self.inner.clone(); + let cache_dir = self.cache_dir.clone(); + let max_cache_bytes = self.max_cache_bytes; + let current_size = self.current_size.clone(); + Box::pin(async move { + // Check cache + { + let mut idx = index.lock().await; + if idx.get(&hash).is_some() { + if let Ok(file) = fs::File::open(&cached).await { + let stream: BlobStream = + Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)); + return Ok(stream); + } + // Cache entry stale — remove + if let Some(entry) = idx.pop(&hash) { + current_size.fetch_sub(entry.size, Ordering::Relaxed); + } + } + } + + // Cache miss — fetch from inner, spool to cache + let self_ref = CachedRef { + cache_dir, + max_cache_bytes, + index: index.clone(), + current_size: current_size.clone(), + }; + let dest = self_ref.fetch_and_cache_static(&hash, &*inner).await?; + let file = fs::File::open(&dest).await.map_err(|e| { + DomainError::internal_error("BlobCache", format!("re-open cached: {e}")) + })?; + let stream: BlobStream = Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)); + Ok(stream) + }) + } + + fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> Pin> + Send + '_>> + { + let hash = hash.to_string(); + let cached = self.cached_path(&hash); + let index = self.index.clone(); + let inner = self.inner.clone(); + let cache_dir = self.cache_dir.clone(); + let max_cache_bytes = self.max_cache_bytes; + let current_size = self.current_size.clone(); + Box::pin(async move { + // Try cache first + { + let mut idx = index.lock().await; + if idx.get(&hash).is_some() { + if let Ok(mut file) = fs::File::open(&cached).await { + file.seek(std::io::SeekFrom::Start(start)) + .await + .map_err(|e| { + DomainError::internal_error("BlobCache", format!("seek: {e}")) + })?; + let take_len = end.map(|e| e - start + 1).unwrap_or(u64::MAX); + let limited = file.take(take_len); + let stream: BlobStream = + Box::pin(ReaderStream::with_capacity(limited, STREAM_CHUNK_SIZE)); + return Ok(stream); + } + if let Some(entry) = idx.pop(&hash) { + current_size.fetch_sub(entry.size, Ordering::Relaxed); + } + } + } + + // Cache miss — fetch full blob into cache, then serve range + let self_ref = CachedRef { + cache_dir, + max_cache_bytes, + index: index.clone(), + current_size: current_size.clone(), + }; + let dest = self_ref.fetch_and_cache_static(&hash, &*inner).await?; + let mut file = fs::File::open(&dest) + .await + .map_err(|e| DomainError::internal_error("BlobCache", format!("re-open: {e}")))?; + file.seek(std::io::SeekFrom::Start(start)) + .await + .map_err(|e| DomainError::internal_error("BlobCache", format!("seek: {e}")))?; + let take_len = end.map(|e| e - start + 1).unwrap_or(u64::MAX); + let limited = file.take(take_len); + let stream: BlobStream = + Box::pin(ReaderStream::with_capacity(limited, STREAM_CHUNK_SIZE)); + Ok(stream) + }) + } + + fn delete_blob( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let hash = hash.to_string(); + let cached = self.cached_path(&hash); + let index = self.index.clone(); + let current_size = self.current_size.clone(); + Box::pin(async move { + inner.delete_blob(&hash).await?; + // Remove from cache + let mut idx = index.lock().await; + if let Some(entry) = idx.pop(&hash) { + current_size.fetch_sub(entry.size, Ordering::Relaxed); + } + let _ = fs::remove_file(&cached).await; + Ok(()) + }) + } + + fn blob_exists( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let hash = hash.to_string(); + let index = self.index.clone(); + Box::pin(async move { + // Check cache first (fast) + { + let mut idx = index.lock().await; + if idx.get(&hash).is_some() { + return Ok(true); + } + } + inner.blob_exists(&hash).await + }) + } + + fn blob_size( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let hash = hash.to_string(); + let index = self.index.clone(); + let cached = self.cached_path(&hash); + Box::pin(async move { + // Check cache + { + let mut idx = index.lock().await; + if let Some(entry) = idx.get(&hash) { + return Ok(entry.size); + } + } + // Fallback to cached file on disk (in case index was lost) + if let Ok(meta) = fs::metadata(&cached).await { + return Ok(meta.len()); + } + inner.blob_size(&hash).await + }) + } + + fn health_check( + &self, + ) -> Pin< + Box> + Send + '_>, + > { + let inner = self.inner.clone(); + let cache_dir = self.cache_dir.clone(); + let current_size = self.current_size.clone(); + let max_bytes = self.max_cache_bytes; + Box::pin(async move { + let mut status = inner.health_check().await?; + let used = current_size.load(Ordering::Relaxed); + status.message = format!( + "{} | Cache: {}/{} bytes used at {}", + status.message, + used, + max_bytes, + cache_dir.display() + ); + status.backend_type = format!("cached({})", status.backend_type); + Ok(status) + }) + } + + fn backend_type(&self) -> &'static str { + "cached" + } + + fn local_blob_path(&self, hash: &str) -> Option { + // If the blob is cached locally, return that path + let path = self.cached_path(hash); + if path.exists() { Some(path) } else { None } + } +} + +// ── Helper struct for owned references in async closures ─────────── + +/// Cloneable set of cache internals — avoids borrow issues in boxed futures. +struct CachedRef { + cache_dir: PathBuf, + max_cache_bytes: u64, + index: Arc>>, + current_size: Arc, +} + +impl CachedRef { + fn cached_path(&self, hash: &str) -> PathBuf { + let prefix = &hash[..2.min(hash.len())]; + self.cache_dir.join(prefix).join(format!("{hash}.blob")) + } + + async fn insert_into_cache_static( + &self, + hash: &str, + source_path: &Path, + ) -> Result<(), DomainError> { + let dest = self.cached_path(hash); + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent).await.map_err(|e| { + DomainError::internal_error("BlobCache", format!("mkdir failed: {e}")) + })?; + } + + let size = fs::metadata(source_path) + .await + .map(|m| m.len()) + .unwrap_or(0); + + fs::copy(source_path, &dest).await.map_err(|e| { + DomainError::internal_error("BlobCache", format!("cache copy failed: {e}")) + })?; + + let mut idx = self.index.lock().await; + if let Some(old) = idx.put(hash.to_string(), CacheEntry { size }) { + self.current_size.fetch_sub(old.size, Ordering::Relaxed); + } + self.current_size.fetch_add(size, Ordering::Relaxed); + + while self.current_size.load(Ordering::Relaxed) > self.max_cache_bytes { + if let Some((evicted_hash, evicted_entry)) = idx.pop_lru() { + self.current_size + .fetch_sub(evicted_entry.size, Ordering::Relaxed); + let evicted_path = self.cached_path(&evicted_hash); + let _ = fs::remove_file(&evicted_path).await; + } else { + break; + } + } + Ok(()) + } + + async fn fetch_and_cache_static( + &self, + hash: &str, + inner: &dyn BlobStorageBackend, + ) -> Result { + let stream = inner.get_blob_stream(hash).await?; + + let dest = self.cached_path(hash); + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent).await.map_err(|e| { + DomainError::internal_error("BlobCache", format!("mkdir failed: {e}")) + })?; + } + + let tmp = dest.with_extension("tmp"); + let mut file = fs::File::create(&tmp) + .await + .map_err(|e| DomainError::internal_error("BlobCache", format!("create tmp: {e}")))?; + + use futures::StreamExt; + let mut stream = stream; + let mut total = 0u64; + while let Some(chunk) = stream.next().await { + let bytes = chunk.map_err(|e| { + DomainError::internal_error("BlobCache", format!("stream read: {e}")) + })?; + total += bytes.len() as u64; + file.write_all(&bytes) + .await + .map_err(|e| DomainError::internal_error("BlobCache", format!("write: {e}")))?; + } + file.flush() + .await + .map_err(|e| DomainError::internal_error("BlobCache", format!("flush: {e}")))?; + drop(file); + + fs::rename(&tmp, &dest) + .await + .map_err(|e| DomainError::internal_error("BlobCache", format!("rename: {e}")))?; + + let mut idx = self.index.lock().await; + if let Some(old) = idx.put(hash.to_string(), CacheEntry { size: total }) { + self.current_size.fetch_sub(old.size, Ordering::Relaxed); + } + self.current_size.fetch_add(total, Ordering::Relaxed); + + while self.current_size.load(Ordering::Relaxed) > self.max_cache_bytes { + if let Some((evicted_hash, evicted_entry)) = idx.pop_lru() { + self.current_size + .fetch_sub(evicted_entry.size, Ordering::Relaxed); + let evicted_path = self.cached_path(&evicted_hash); + let _ = fs::remove_file(&evicted_path).await; + } else { + break; + } + } + + Ok(dest) + } +} diff --git a/src/infrastructure/services/dedup_service.rs b/src/infrastructure/services/dedup_service.rs index 34dc96e5..88c55f07 100644 --- a/src/infrastructure/services/dedup_service.rs +++ b/src/infrastructure/services/dedup_service.rs @@ -39,24 +39,22 @@ use sqlx::PgPool; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; -use tokio::fs::{self, File}; -use tokio::io::{AsyncReadExt, AsyncSeekExt}; -use tokio_util::io::ReaderStream; +use tokio::fs; +use crate::application::ports::blob_storage_ports::BlobStorageBackend; use crate::application::ports::dedup_ports::{ BlobMetadataDto, DedupPort, DedupResultDto, DedupStatsDto, }; use crate::domain::errors::{DomainError, ErrorKind}; -/// Chunk size for streaming file reads (256 KB) -const STREAM_CHUNK_SIZE: usize = 256 * 1024; - /// Content-Addressable Storage Service (PostgreSQL-backed) +/// +/// Delegates all byte-level I/O to a [`BlobStorageBackend`] implementation +/// (local filesystem, S3, etc.) while keeping BLAKE3 hashing, ref-counting +/// and the PostgreSQL dedup index here. pub struct DedupService { - /// Root directory for blob storage on the filesystem - blob_root: PathBuf, - /// Root directory for temporary files during upload - temp_root: PathBuf, + /// Pluggable blob storage backend (local FS, S3, …). + backend: Arc, /// PostgreSQL connection pool (dedup index in `storage.blobs`) — primary, /// used by request-path operations (store_from_file, etc.). pool: Arc, @@ -65,39 +63,19 @@ pub struct DedupService { maintenance_pool: Arc, } -/// Compile-time lookup table for the 256 two-digit lowercase hex prefixes ("00"…"ff"). -/// Avoids a `format!("{:02x}", i)` allocation on every iteration of `initialize()`. -static HEX_PREFIXES: [&str; 256] = [ - "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0a", "0b", "0c", "0d", "0e", "0f", - "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "1a", "1b", "1c", "1d", "1e", "1f", - "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "2a", "2b", "2c", "2d", "2e", "2f", - "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "3a", "3b", "3c", "3d", "3e", "3f", - "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "4a", "4b", "4c", "4d", "4e", "4f", - "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "5a", "5b", "5c", "5d", "5e", "5f", - "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "6a", "6b", "6c", "6d", "6e", "6f", - "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "7a", "7b", "7c", "7d", "7e", "7f", - "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "8a", "8b", "8c", "8d", "8e", "8f", - "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "9a", "9b", "9c", "9d", "9e", "9f", - "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9", "aa", "ab", "ac", "ad", "ae", "af", - "b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9", "ba", "bb", "bc", "bd", "be", "bf", - "c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "ca", "cb", "cc", "cd", "ce", "cf", - "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "da", "db", "dc", "dd", "de", "df", - "e0", "e1", "e2", "e3", "e4", "e5", "e6", "e7", "e8", "e9", "ea", "eb", "ec", "ed", "ee", "ef", - "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "fa", "fb", "fc", "fd", "fe", "ff", -]; - impl DedupService { /// Create a new dedup service backed by PostgreSQL. /// + /// * `backend` — pluggable blob storage (local filesystem, S3, etc.). /// * `pool` — primary pool for request-path operations. /// * `maintenance_pool` — isolated pool for verify_integrity / garbage_collect. - pub fn new(storage_root: &Path, pool: Arc, maintenance_pool: Arc) -> Self { - let blob_root = storage_root.join(".blobs"); - let temp_root = storage_root.join(".dedup_temp"); - + pub fn new( + backend: Arc, + pool: Arc, + maintenance_pool: Arc, + ) -> Self { Self { - blob_root, - temp_root, + backend, pool, maintenance_pool, } @@ -106,6 +84,7 @@ impl DedupService { /// Creates a stub instance for testing — never hits PG or the filesystem. #[cfg(any(test, feature = "integration_tests"))] pub fn new_stub() -> Self { + use crate::infrastructure::services::local_blob_backend::LocalBlobBackend; let stub_pool = Arc::new( sqlx::pool::PoolOptions::::new() .max_connections(1) @@ -113,29 +92,15 @@ impl DedupService { .unwrap(), ); Self { - blob_root: std::path::PathBuf::from("/tmp/oxicloud_stub_blobs"), - temp_root: std::path::PathBuf::from("/tmp/oxicloud_stub_temp"), + backend: Arc::new(LocalBlobBackend::new(Path::new("/tmp/oxicloud_stub_blobs"))), pool: stub_pool.clone(), maintenance_pool: stub_pool, } } - /// Initialize the service (create blob directories on the filesystem). + /// Initialize the service (delegate to backend + log stats from PG). pub async fn initialize(&self) -> Result<(), DomainError> { - // Create directories - fs::create_dir_all(&self.blob_root) - .await - .map_err(DomainError::from)?; - fs::create_dir_all(&self.temp_root) - .await - .map_err(DomainError::from)?; - - // Create hash prefix directories (00-ff) - for prefix in &HEX_PREFIXES { - fs::create_dir_all(self.blob_root.join(prefix)) - .await - .map_err(DomainError::from)?; - } + self.backend.initialize().await?; // Log existing blob stats from PG let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM storage.blobs") @@ -150,7 +115,8 @@ impl DedupService { .unwrap_or(0); tracing::info!( - "Dedup service initialized (PostgreSQL-backed): {} blobs, {} bytes stored", + "Dedup service initialized (backend={}): {} blobs, {} bytes stored", + self.backend.backend_type(), count, total_bytes ); @@ -158,12 +124,18 @@ impl DedupService { Ok(()) } + /// Return a reference to the underlying blob storage backend. + pub fn backend(&self) -> &Arc { + &self.backend + } + // ── Path helpers ───────────────────────────────────────────── - /// Get the blob path for a given hash. + /// Get the local blob path for a given hash (if the backend supports it). pub fn blob_path(&self, hash: &str) -> PathBuf { - let prefix = &hash[0..2]; - self.blob_root.join(prefix).join(format!("{}.blob", hash)) + self.backend + .local_blob_path(hash) + .unwrap_or_else(|| PathBuf::from(format!("remote://{}", hash))) } // ── Hash helpers ───────────────────────────────────────────── @@ -193,9 +165,9 @@ impl DedupService { /// Store content with deduplication (streaming from file). /// - /// **Write-first strategy**: the source file is moved/copied to the - /// blob store *before* touching PostgreSQL, so the PG connection is - /// never held during disk I/O. + /// **Write-first strategy**: the source file is moved/uploaded to the + /// blob backend *before* touching PostgreSQL, so the PG connection is + /// never held during I/O. /// /// If `pre_computed_hash` is `Some`, the file will NOT be re-read for /// BLAKE3 — saving one full sequential read (the biggest I/O win). @@ -205,13 +177,6 @@ impl DedupService { content_type: Option, pre_computed_hash: Option, ) -> Result { - let file_size = fs::metadata(source_path) - .await - .map_err(|e| { - DomainError::internal_error("Dedup", format!("Failed to get file metadata: {}", e)) - })? - .len(); - // Use pre-computed hash if available, otherwise calculate (streaming) let hash = match pre_computed_hash { Some(h) => h, @@ -220,44 +185,11 @@ impl DedupService { .map_err(DomainError::from)?, }; + // ── Phase 1: Place blob in backend (NO PG connection held) ─── + let file_size = self.backend.put_blob(&hash, source_path).await?; + let blob_path = self.blob_path(&hash); - // ── Phase 1: Move/place blob on disk (NO PG connection held) ─ - // - // If the blob file already exists on disk, the source is simply - // deleted — the file content is identical by definition. - if fs::try_exists(&blob_path).await.unwrap_or(false) { - // Blob already on disk — discard the source file - let _ = fs::remove_file(source_path).await; - } else { - // Parent directory (xx/) guaranteed to exist — created by initialize() - - // rename is atomic on the same filesystem. If source and blob - // dirs live on different filesystems (rare), this falls back to - // copy+delete which is slower but still correct. - if let Err(e) = fs::rename(source_path, &blob_path).await { - if e.raw_os_error() == Some(18) { - // EXDEV: cross-device link — fall back to copy+delete - fs::copy(source_path, &blob_path).await.map_err(|ce| { - DomainError::internal_error( - "Dedup", - format!("Failed to copy file to blob store: {}", ce), - ) - })?; - let _ = fs::remove_file(source_path).await; - } else if fs::try_exists(&blob_path).await.unwrap_or(false) { - // Another writer may have placed the blob concurrently - let _ = fs::remove_file(source_path).await; - tracing::debug!("Blob file placed by concurrent writer: {}", e); - } else { - return Err(DomainError::internal_error( - "Dedup", - format!("Failed to move file to blob store: {}", e), - )); - } - } - } - // ── Phase 2: Single atomic upsert (~2-4 ms, no explicit TX) ─ let ref_count: i32 = sqlx::query_scalar( "INSERT INTO storage.blobs (hash, size, ref_count, content_type) @@ -415,10 +347,9 @@ impl DedupService { DomainError::internal_error("Dedup", format!("Failed to commit: {}", e)) })?; - // Delete blob file AFTER committing PG — the row is gone, so no - // concurrent store_from_file can resurrect a reference to this hash. - let blob_path = self.blob_path(hash); - if let Err(e) = fs::remove_file(&blob_path).await { + // Delete blob from backend AFTER committing PG — the row is gone, + // so no concurrent store_from_file can resurrect a reference. + if let Err(e) = self.backend.delete_blob(hash).await { tracing::warn!("Failed to delete blob file {}: {}", hash, e); } @@ -449,31 +380,16 @@ impl DedupService { // ── Read operations ────────────────────────────────────────── - /// Stream blob content in 64 KB chunks — constant memory (~64 KB per stream). - /// - /// A 1 GB file uses the same ~64 KB as a 1 KB file. + /// Stream blob content in chunks — constant memory usage. pub async fn read_blob_stream( &self, hash: &str, ) -> Result> + Send>>, DomainError> { - let blob_path = self.blob_path(hash); - let file = File::open(&blob_path).await.map_err(|e| { - DomainError::new( - ErrorKind::NotFound, - "Blob", - format!("Failed to open blob {}: {}", hash, e), - ) - })?; - Ok(Box::pin(ReaderStream::with_capacity( - file, - STREAM_CHUNK_SIZE, - ))) + self.backend.get_blob_stream(hash).await } /// Stream a byte range of a blob — only reads the requested portion. - /// - /// Uses seek + take so a 1 MB range request on a 1 GB file only reads 1 MB. pub async fn read_blob_range_stream( &self, hash: &str, @@ -481,49 +397,12 @@ impl DedupService { end: Option, ) -> Result> + Send>>, DomainError> { - let blob_path = self.blob_path(hash); - let mut file = File::open(&blob_path).await.map_err(|e| { - DomainError::new( - ErrorKind::NotFound, - "Blob", - format!("Failed to open blob {}: {}", hash, e), - ) - })?; - - // Seek to the start position - file.seek(std::io::SeekFrom::Start(start)) - .await - .map_err(|e| { - DomainError::internal_error("Blob", format!("Failed to seek in blob: {}", e)) - })?; - - // If an end is specified, limit the read with take() - if let Some(end_pos) = end { - let limit = end_pos.saturating_sub(start); - let limited = file.take(limit); - Ok(Box::pin(ReaderStream::with_capacity( - limited, - STREAM_CHUNK_SIZE, - ))) - } else { - Ok(Box::pin(ReaderStream::with_capacity( - file, - STREAM_CHUNK_SIZE, - ))) - } + self.backend.get_blob_range_stream(hash, start, end).await } /// Get the size of a blob without reading its content. pub async fn blob_size(&self, hash: &str) -> Result { - let blob_path = self.blob_path(hash); - let meta = fs::metadata(&blob_path).await.map_err(|e| { - DomainError::new( - ErrorKind::NotFound, - "Blob", - format!("Failed to stat blob {}: {}", hash, e), - ) - })?; - Ok(meta.len()) + self.backend.blob_size(hash).await } // ── Statistics (computed from PG) ──────────────────────────── @@ -596,56 +475,46 @@ impl DedupService { // Flush when batch is full or we've exhausted the cursor if batch.len() >= VERIFY_CONCURRENCY || (is_done && !batch.is_empty()) { - let blob_root = self.blob_root.clone(); + let backend = self.backend.clone(); let current_batch = std::mem::replace(&mut batch, Vec::with_capacity(VERIFY_CONCURRENCY)); let issues: Vec = stream::iter(current_batch) .map(move |(hash, expected_size)| { - let blob_root = blob_root.clone(); + let backend = backend.clone(); async move { - let prefix = &hash[0..2]; - let blob_path = blob_root.join(prefix).join(format!("{}.blob", hash)); - let mut issues = Vec::new(); - // Single async metadata() replaces the previous - // blocking .exists() + separate metadata() — one - // stat() syscall instead of two, and non-blocking. - let file_meta = match fs::metadata(&blob_path).await { - Ok(m) => m, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - issues.push(format!("{}: file missing on disk", hash)); - return issues; + // Check existence + size via backend + match backend.blob_size(&hash).await { + Ok(actual_size) => { + if actual_size != expected_size as u64 { + issues.push(format!( + "{}: size mismatch (expected: {}, actual: {})", + hash, expected_size, actual_size, + )); + } } - Err(e) => { - issues.push(format!("{}: metadata error ({})", hash, e)); + Err(_) => { + issues.push(format!("{}: blob missing in backend", hash)); return issues; } }; - // Check size - if file_meta.len() != expected_size as u64 { - issues.push(format!( - "{}: size mismatch (expected: {}, actual: {})", - hash, - expected_size, - file_meta.len(), - )); - } - - // Verify hash - match Self::hash_file(&blob_path).await { - Ok(actual_hash) => { - if actual_hash != hash { - issues.push(format!( - "{}: hash mismatch (actual: {})", - hash, actual_hash, - )); + // Verify hash — only possible for local backends + if let Some(blob_path) = backend.local_blob_path(&hash) { + match Self::hash_file(&blob_path).await { + Ok(actual_hash) => { + if actual_hash != hash { + issues.push(format!( + "{}: hash mismatch (actual: {})", + hash, actual_hash, + )); + } + } + Err(e) => { + issues.push(format!("{}: read error ({})", hash, e)); } - } - Err(e) => { - issues.push(format!("{}: read error ({})", hash, e)); } } @@ -717,20 +586,20 @@ impl DedupService { // Also clean up any thumbnail files for these blob hashes // (thumbnails are keyed by blob_hash and live under // storage_root/.thumbnails/{icon,preview,large}/{hash}.jpg). - let thumbnails_root = self - .blob_root - .parent() - .unwrap_or(&self.blob_root) - .join(".thumbnails"); + for (hash, size) in &batch { - let blob_path = self.blob_path(hash); - if let Err(e) = fs::remove_file(&blob_path).await { + if let Err(e) = self.backend.delete_blob(hash).await { tracing::warn!("Failed to delete orphan blob file {hash}: {e}"); } - // Remove associated thumbnail files (best-effort) - for dir in &["icon", "preview", "large"] { - let thumb = thumbnails_root.join(dir).join(format!("{hash}.jpg")); - let _ = fs::remove_file(&thumb).await; + // Remove associated thumbnail files (best-effort, always local) + if let Some(blob_path) = self.backend.local_blob_path(hash) + && let Some(storage_root) = blob_path.ancestors().nth(3) + { + let thumbnails_root = storage_root.join(".thumbnails"); + for dir in &["icon", "preview", "large"] { + let thumb = thumbnails_root.join(dir).join(format!("{hash}.jpg")); + let _ = fs::remove_file(&thumb).await; + } } total_bytes += *size as u64; } diff --git a/src/infrastructure/services/encrypted_blob_backend.rs b/src/infrastructure/services/encrypted_blob_backend.rs new file mode 100644 index 00000000..4b0bdcab --- /dev/null +++ b/src/infrastructure/services/encrypted_blob_backend.rs @@ -0,0 +1,298 @@ +//! `EncryptedBlobBackend` — AES-256-GCM encryption decorator for blob storage. +//! +//! Transparently encrypts blobs before they reach the inner backend and +//! decrypts them on read. Each blob gets a random 96-bit nonce which is +//! prepended to the ciphertext. The GCM authentication tag (16 bytes) is +//! appended by the cipher. +//! +//! **IMPORTANT**: BLAKE3 hashing is performed on the *plaintext* by +//! `DedupService` before this layer sees the blob, so content-addressable +//! dedup still works correctly. +//! +//! Layout on disk/S3: `[12-byte nonce][ciphertext + 16-byte GCM tag]` + +use std::path::{Path, PathBuf}; +use std::pin::Pin; + +use aes_gcm::aead::{Aead, KeyInit, OsRng}; +use aes_gcm::{AeadCore, Aes256Gcm, Nonce}; +use bytes::Bytes; +use std::sync::Arc; +use tokio::fs; +use tokio::io::AsyncWriteExt; + +use crate::application::ports::blob_storage_ports::{ + BlobStorageBackend, BlobStream, StorageHealthStatus, +}; +use crate::domain::errors::DomainError; + +/// Nonce size for AES-256-GCM (96 bits = 12 bytes). +const NONCE_SIZE: usize = 12; + +/// `BlobStorageBackend` decorator that encrypts blobs at rest. +pub struct EncryptedBlobBackend { + inner: Arc, + cipher: Aes256Gcm, +} + +impl EncryptedBlobBackend { + /// Create a new encryption layer wrapping `inner`. + /// + /// `key` must be exactly 32 bytes (AES-256). + pub fn new(inner: Arc, key: &[u8; 32]) -> Self { + let cipher = Aes256Gcm::new_from_slice(key).expect("AES-256 key must be 32 bytes"); + Self { inner, cipher } + } + + /// Generate a random 32-byte key suitable for AES-256. + pub fn generate_key() -> [u8; 32] { + use aes_gcm::aead::rand_core::RngCore; + let mut key = [0u8; 32]; + OsRng.fill_bytes(&mut key); + key + } +} + +impl BlobStorageBackend for EncryptedBlobBackend { + fn initialize( + &self, + ) -> Pin> + Send + '_>> { + self.inner.initialize() + } + + fn put_blob( + &self, + hash: &str, + source_path: &Path, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let hash = hash.to_string(); + let source = source_path.to_path_buf(); + // Clone cipher key material (Aes256Gcm is not Send-safe to move across await) + let cipher = self.cipher.clone(); + Box::pin(async move { + // Read plaintext from source + let plaintext = fs::read(&source).await.map_err(|e| { + DomainError::internal_error("Encryption", format!("read source: {e}")) + })?; + + // Encrypt: nonce || ciphertext (includes GCM tag) + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let ciphertext = cipher.encrypt(&nonce, plaintext.as_ref()).map_err(|e| { + DomainError::internal_error("Encryption", format!("encrypt failed: {e}")) + })?; + + // Write encrypted blob to a temp file + let tmp = source.with_extension("enc.tmp"); + let mut file = fs::File::create(&tmp).await.map_err(|e| { + DomainError::internal_error("Encryption", format!("create tmp: {e}")) + })?; + file.write_all(nonce.as_slice()).await.map_err(|e| { + DomainError::internal_error("Encryption", format!("write nonce: {e}")) + })?; + file.write_all(&ciphertext).await.map_err(|e| { + DomainError::internal_error("Encryption", format!("write ciphertext: {e}")) + })?; + file.flush() + .await + .map_err(|e| DomainError::internal_error("Encryption", format!("flush: {e}")))?; + drop(file); + + let result = inner.put_blob(&hash, &tmp).await; + let _ = fs::remove_file(&tmp).await; + result + }) + } + + fn get_blob_stream( + &self, + hash: &str, + ) -> Pin> + Send + '_>> + { + let inner = self.inner.clone(); + let hash = hash.to_string(); + let cipher = self.cipher.clone(); + Box::pin(async move { + // Read entire encrypted blob (nonce + ciphertext) into memory for decryption + let enc_stream = inner.get_blob_stream(&hash).await?; + let encrypted = collect_stream(enc_stream).await?; + + if encrypted.len() < NONCE_SIZE { + return Err(DomainError::internal_error( + "Encryption", + "encrypted blob too short (missing nonce)", + )); + } + + let (nonce_bytes, ciphertext) = encrypted.split_at(NONCE_SIZE); + let nonce = Nonce::from_slice(nonce_bytes); + let plaintext = cipher.decrypt(nonce, ciphertext).map_err(|e| { + DomainError::internal_error("Encryption", format!("decrypt failed: {e}")) + })?; + + let stream: BlobStream = + Box::pin(futures::stream::once( + async move { Ok(Bytes::from(plaintext)) }, + )); + Ok(stream) + }) + } + + fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> Pin> + Send + '_>> + { + let inner = self.inner.clone(); + let hash = hash.to_string(); + let cipher = self.cipher.clone(); + Box::pin(async move { + // Must decrypt the full blob then slice the plaintext range + let enc_stream = inner.get_blob_stream(&hash).await?; + let encrypted = collect_stream(enc_stream).await?; + + if encrypted.len() < NONCE_SIZE { + return Err(DomainError::internal_error( + "Encryption", + "encrypted blob too short", + )); + } + + let (nonce_bytes, ciphertext) = encrypted.split_at(NONCE_SIZE); + let nonce = Nonce::from_slice(nonce_bytes); + let plaintext = cipher.decrypt(nonce, ciphertext).map_err(|e| { + DomainError::internal_error("Encryption", format!("decrypt failed: {e}")) + })?; + + let start = start as usize; + let end = end.map(|e| (e as usize) + 1).unwrap_or(plaintext.len()); + let end = end.min(plaintext.len()); + let start = start.min(end); + + let slice = Bytes::from(plaintext[start..end].to_vec()); + let stream: BlobStream = Box::pin(futures::stream::once(async move { Ok(slice) })); + Ok(stream) + }) + } + + fn delete_blob( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + self.inner.delete_blob(hash) + } + + fn blob_exists( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + self.inner.blob_exists(hash) + } + + fn blob_size( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + // The stored size includes nonce + GCM tag overhead. + // Return the *plaintext* size by subtracting overhead. + let inner = self.inner.clone(); + let hash = hash.to_string(); + Box::pin(async move { + let encrypted_size = inner.blob_size(&hash).await?; + // overhead = 12 (nonce) + 16 (GCM tag) = 28 bytes + Ok(encrypted_size.saturating_sub(28)) + }) + } + + fn health_check( + &self, + ) -> Pin< + Box> + Send + '_>, + > { + let inner = self.inner.clone(); + Box::pin(async move { + let mut status = inner.health_check().await?; + status.backend_type = format!("encrypted({})", status.backend_type); + status.message = format!("{} | Encryption: AES-256-GCM", status.message); + Ok(status) + }) + } + + fn backend_type(&self) -> &'static str { + "encrypted" + } + + fn local_blob_path(&self, _hash: &str) -> Option { + // Encrypted blobs cannot be served directly from disk + None + } +} + +/// Collect a byte stream into a single `Vec`. +async fn collect_stream(stream: BlobStream) -> Result, DomainError> { + use futures::StreamExt; + let mut stream = stream; + let mut buf = Vec::new(); + while let Some(chunk) = stream.next().await { + let bytes = chunk + .map_err(|e| DomainError::internal_error("Encryption", format!("stream read: {e}")))?; + buf.extend_from_slice(&bytes); + } + Ok(buf) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::infrastructure::services::local_blob_backend::LocalBlobBackend; + use tempfile::TempDir; + use tokio::io::AsyncWriteExt; + + #[tokio::test] + async fn test_encrypt_decrypt_roundtrip() { + let tmp = TempDir::new().unwrap(); + let blob_dir = tmp.path().join("blobs"); + let local = Arc::new(LocalBlobBackend::new(&blob_dir)); + local.initialize().await.unwrap(); + + let key = EncryptedBlobBackend::generate_key(); + let encrypted = EncryptedBlobBackend::new(local, &key); + + // Write a test blob + let data = b"Hello, encrypted world!"; + let source = tmp.path().join("test.tmp"); + let mut f = fs::File::create(&source).await.unwrap(); + f.write_all(data).await.unwrap(); + f.flush().await.unwrap(); + drop(f); + + let hash = "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"; + encrypted.put_blob(hash, &source).await.unwrap(); + + // Read back via stream + let stream = encrypted.get_blob_stream(hash).await.unwrap(); + let decrypted = collect_stream(stream).await.unwrap(); + assert_eq!(decrypted, data); + + // Read range + let range_stream = encrypted + .get_blob_range_stream(hash, 7, Some(15)) + .await + .unwrap(); + let range_data = collect_stream(range_stream).await.unwrap(); + assert_eq!(range_data, b"encrypted"); + + // Size should reflect plaintext + let size = encrypted.blob_size(hash).await.unwrap(); + assert_eq!(size, data.len() as u64); + + // Exists + assert!(encrypted.blob_exists(hash).await.unwrap()); + + // Delete + encrypted.delete_blob(hash).await.unwrap(); + assert!(!encrypted.blob_exists(hash).await.unwrap()); + } +} diff --git a/src/infrastructure/services/local_blob_backend.rs b/src/infrastructure/services/local_blob_backend.rs new file mode 100644 index 00000000..86c7e536 --- /dev/null +++ b/src/infrastructure/services/local_blob_backend.rs @@ -0,0 +1,277 @@ +//! Local Filesystem Blob Backend — stores blobs under `.blobs/{prefix}/{hash}.blob`. +//! +//! This is the default backend and a direct extraction of the filesystem I/O +//! that previously lived inside `DedupService`. + +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use tokio::fs::{self, File}; +use tokio::io::AsyncSeekExt; +use tokio_util::io::ReaderStream; + +use crate::application::ports::blob_storage_ports::{ + BlobStorageBackend, BlobStream, StorageHealthStatus, +}; +use crate::domain::errors::{DomainError, ErrorKind}; + +/// Chunk size for streaming file reads (256 KB). +const STREAM_CHUNK_SIZE: usize = 256 * 1024; + +/// Compile-time lookup table for the 256 two-digit lowercase hex prefixes ("00"…"ff"). +static HEX_PREFIXES: [&str; 256] = [ + "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0a", "0b", "0c", "0d", "0e", "0f", + "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "1a", "1b", "1c", "1d", "1e", "1f", + "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "2a", "2b", "2c", "2d", "2e", "2f", + "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "3a", "3b", "3c", "3d", "3e", "3f", + "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "4a", "4b", "4c", "4d", "4e", "4f", + "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "5a", "5b", "5c", "5d", "5e", "5f", + "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "6a", "6b", "6c", "6d", "6e", "6f", + "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "7a", "7b", "7c", "7d", "7e", "7f", + "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "8a", "8b", "8c", "8d", "8e", "8f", + "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "9a", "9b", "9c", "9d", "9e", "9f", + "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9", "aa", "ab", "ac", "ad", "ae", "af", + "b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9", "ba", "bb", "bc", "bd", "be", "bf", + "c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "ca", "cb", "cc", "cd", "ce", "cf", + "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "da", "db", "dc", "dd", "de", "df", + "e0", "e1", "e2", "e3", "e4", "e5", "e6", "e7", "e8", "e9", "ea", "eb", "ec", "ed", "ee", "ef", + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "fa", "fb", "fc", "fd", "fe", "ff", +]; + +/// Local filesystem blob backend. +/// +/// Blobs are stored under `blob_root/{2-char-prefix}/{hash}.blob`. +/// Temporary upload staging uses `temp_root/`. +pub struct LocalBlobBackend { + blob_root: PathBuf, + temp_root: PathBuf, +} + +impl LocalBlobBackend { + /// Create a new local backend rooted at `storage_root`. + /// + /// Blob files go under `{storage_root}/.blobs/`, temp files under + /// `{storage_root}/.dedup_temp/`. + pub fn new(storage_root: &Path) -> Self { + Self { + blob_root: storage_root.join(".blobs"), + temp_root: storage_root.join(".dedup_temp"), + } + } + + /// Compute the filesystem path for a blob hash. + pub fn blob_path(&self, hash: &str) -> PathBuf { + let prefix = &hash[0..2]; + self.blob_root.join(prefix).join(format!("{}.blob", hash)) + } + + /// Return a reference to the blob root directory. + pub fn blob_root(&self) -> &Path { + &self.blob_root + } +} + +impl BlobStorageBackend for LocalBlobBackend { + fn initialize( + &self, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + fs::create_dir_all(&self.blob_root) + .await + .map_err(DomainError::from)?; + fs::create_dir_all(&self.temp_root) + .await + .map_err(DomainError::from)?; + + // Create the 256 hash-prefix directories (00-ff) + for prefix in &HEX_PREFIXES { + fs::create_dir_all(self.blob_root.join(prefix)) + .await + .map_err(DomainError::from)?; + } + Ok(()) + }) + } + + fn put_blob( + &self, + hash: &str, + source_path: &Path, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + let source_path = source_path.to_owned(); + Box::pin(async move { + let blob_path = self.blob_path(&hash); + + let file_size = fs::metadata(&source_path) + .await + .map_err(|e| { + DomainError::internal_error( + "Blob", + format!("Failed to stat source file: {}", e), + ) + })? + .len(); + + // Idempotent: if blob already exists, just remove the source + if fs::try_exists(&blob_path).await.unwrap_or(false) { + let _ = fs::remove_file(&source_path).await; + return Ok(file_size); + } + + // Atomic rename (same filesystem). Falls back to copy+delete for + // cross-device moves (EXDEV errno 18). + if let Err(e) = fs::rename(&source_path, &blob_path).await { + if e.raw_os_error() == Some(18) { + // EXDEV — cross-device link + fs::copy(&source_path, &blob_path).await.map_err(|ce| { + DomainError::internal_error( + "Blob", + format!("Failed to copy file to blob store: {}", ce), + ) + })?; + let _ = fs::remove_file(&source_path).await; + } else if fs::try_exists(&blob_path).await.unwrap_or(false) { + // Concurrent writer placed the blob — discard our copy + let _ = fs::remove_file(&source_path).await; + tracing::debug!("Blob placed by concurrent writer: {}", e); + } else { + return Err(DomainError::internal_error( + "Blob", + format!("Failed to move file to blob store: {}", e), + )); + } + } + + Ok(file_size) + }) + } + + fn get_blob_stream( + &self, + hash: &str, + ) -> Pin> + Send + '_>> + { + let hash = hash.to_owned(); + Box::pin(async move { + let blob_path = self.blob_path(&hash); + let file = File::open(&blob_path).await.map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "Blob", + format!("Failed to open blob {}: {}", hash, e), + ) + })?; + Ok(Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)) as BlobStream) + }) + } + + fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> Pin> + Send + '_>> + { + let hash = hash.to_owned(); + Box::pin(async move { + let blob_path = self.blob_path(&hash); + let mut file = File::open(&blob_path).await.map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "Blob", + format!("Failed to open blob {}: {}", hash, e), + ) + })?; + + file.seek(std::io::SeekFrom::Start(start)) + .await + .map_err(|e| { + DomainError::internal_error("Blob", format!("Failed to seek in blob: {}", e)) + })?; + + if let Some(end_pos) = end { + use tokio::io::AsyncReadExt; + let limit = end_pos.saturating_sub(start); + let limited = file.take(limit); + Ok(Box::pin(ReaderStream::with_capacity(limited, STREAM_CHUNK_SIZE)) as BlobStream) + } else { + Ok(Box::pin(ReaderStream::with_capacity(file, STREAM_CHUNK_SIZE)) as BlobStream) + } + }) + } + + fn delete_blob( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let blob_path = self.blob_path(&hash); + match fs::remove_file(&blob_path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), // idempotent + Err(e) => Err(DomainError::internal_error( + "Blob", + format!("Failed to delete blob {}: {}", hash, e), + )), + } + }) + } + + fn blob_exists( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let blob_path = self.blob_path(&hash); + Ok(fs::try_exists(&blob_path).await.unwrap_or(false)) + }) + } + + fn blob_size( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let blob_path = self.blob_path(&hash); + let meta = fs::metadata(&blob_path).await.map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "Blob", + format!("Failed to stat blob {}: {}", hash, e), + ) + })?; + Ok(meta.len()) + }) + } + + fn health_check( + &self, + ) -> Pin< + Box> + Send + '_>, + > { + Box::pin(async move { + let writable = fs::metadata(&self.blob_root).await.is_ok(); + Ok(StorageHealthStatus { + connected: writable, + backend_type: "local".to_string(), + message: if writable { + "Local filesystem is accessible".to_string() + } else { + "Blob root directory is not accessible".to_string() + }, + available_bytes: None, + }) + }) + } + + fn backend_type(&self) -> &'static str { + "local" + } + + fn local_blob_path(&self, hash: &str) -> Option { + Some(self.blob_path(hash)) + } +} diff --git a/src/infrastructure/services/migration_blob_backend.rs b/src/infrastructure/services/migration_blob_backend.rs new file mode 100644 index 00000000..1f302fe5 --- /dev/null +++ b/src/infrastructure/services/migration_blob_backend.rs @@ -0,0 +1,204 @@ +//! `MigrationBlobBackend` — decorator that enables zero-downtime migration +//! between blob storage backends. +//! +//! During a migration the decorator writes to the **target** backend and reads +//! from **target-first-then-source** (dual-read). A background job +//! (see `migration_job.rs`) copies remaining blobs in the background. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use serde::Serialize; +use tokio::sync::RwLock; + +use crate::application::ports::blob_storage_ports::{ + BlobStorageBackend, BlobStream, StorageHealthStatus, +}; +use crate::common::errors::DomainError; + +// ── Migration state ──────────────────────────────────────────────── + +/// Progress of an ongoing (or completed) backend migration. +#[derive(Debug, Clone, Serialize)] +pub struct MigrationState { + pub status: MigrationStatus, + pub total_blobs: u64, + pub migrated_blobs: u64, + pub migrated_bytes: u64, + pub failed_blobs: Vec, + pub started_at: Option>, + pub completed_at: Option>, +} + +impl Default for MigrationState { + fn default() -> Self { + Self { + status: MigrationStatus::Idle, + total_blobs: 0, + migrated_blobs: 0, + migrated_bytes: 0, + failed_blobs: Vec::new(), + started_at: None, + completed_at: None, + } + } +} + +/// Status of the migration job. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MigrationStatus { + Idle, + Running, + Paused, + Completed, + Failed, +} + +// ── MigrationBlobBackend ─────────────────────────────────────────── + +/// A `BlobStorageBackend` decorator that proxies requests to a *source* +/// (old) and *target* (new) backend, enabling live migration. +pub struct MigrationBlobBackend { + source: Arc, + target: Arc, + state: Arc>, +} + +impl MigrationBlobBackend { + pub fn new( + source: Arc, + target: Arc, + state: Arc>, + ) -> Self { + Self { + source, + target, + state, + } + } + + pub fn state(&self) -> &Arc> { + &self.state + } + + pub fn source(&self) -> &Arc { + &self.source + } + + pub fn target(&self) -> &Arc { + &self.target + } +} + +/// Boxed future alias (same as in the trait module). +type BoxFut<'a, T> = Pin + Send + 'a>>; + +impl BlobStorageBackend for MigrationBlobBackend { + fn initialize(&self) -> BoxFut<'_, Result<(), DomainError>> { + Box::pin(async move { + self.target.initialize().await?; + // Source is already initialised; call anyway for idempotency. + self.source.initialize().await?; + Ok(()) + }) + } + + /// Writes go to **target** only. + fn put_blob(&self, hash: &str, source_path: &Path) -> BoxFut<'_, Result> { + let hash = hash.to_string(); + let path = source_path.to_path_buf(); + Box::pin(async move { self.target.put_blob(&hash, &path).await }) + } + + /// Read from target first; fall back to source. + fn get_blob_stream(&self, hash: &str) -> BoxFut<'_, Result> { + let hash = hash.to_string(); + Box::pin(async move { + match self.target.get_blob_stream(&hash).await { + Ok(stream) => Ok(stream), + Err(_) => self.source.get_blob_stream(&hash).await, + } + }) + } + + fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> BoxFut<'_, Result> { + let hash = hash.to_string(); + Box::pin(async move { + match self.target.get_blob_range_stream(&hash, start, end).await { + Ok(stream) => Ok(stream), + Err(_) => self.source.get_blob_range_stream(&hash, start, end).await, + } + }) + } + + /// Delete from **both** backends (best-effort on source). + fn delete_blob(&self, hash: &str) -> BoxFut<'_, Result<(), DomainError>> { + let hash = hash.to_string(); + Box::pin(async move { + self.target.delete_blob(&hash).await?; + // Best-effort on source — ignore errors (blob may already be gone). + let _ = self.source.delete_blob(&hash).await; + Ok(()) + }) + } + + /// Exists in either backend. + fn blob_exists(&self, hash: &str) -> BoxFut<'_, Result> { + let hash = hash.to_string(); + Box::pin(async move { + if self.target.blob_exists(&hash).await? { + return Ok(true); + } + self.source.blob_exists(&hash).await + }) + } + + fn blob_size(&self, hash: &str) -> BoxFut<'_, Result> { + let hash = hash.to_string(); + Box::pin(async move { + match self.target.blob_size(&hash).await { + Ok(sz) => Ok(sz), + Err(_) => self.source.blob_size(&hash).await, + } + }) + } + + fn health_check(&self) -> BoxFut<'_, Result> { + Box::pin(async move { + let target_health = self.target.health_check().await?; + let source_health = self.source.health_check().await?; + Ok(StorageHealthStatus { + connected: target_health.connected && source_health.connected, + backend_type: format!( + "migration({} → {})", + source_health.backend_type, target_health.backend_type + ), + message: format!( + "Source: {} | Target: {}", + source_health.message, target_health.message + ), + available_bytes: target_health.available_bytes, + }) + }) + } + + fn backend_type(&self) -> &'static str { + "migration" + } + + fn local_blob_path(&self, hash: &str) -> Option { + // Prefer target, fall back to source. + self.target + .local_blob_path(hash) + .or_else(|| self.source.local_blob_path(hash)) + } +} diff --git a/src/infrastructure/services/migration_job.rs b/src/infrastructure/services/migration_job.rs new file mode 100644 index 00000000..c2cd3673 --- /dev/null +++ b/src/infrastructure/services/migration_job.rs @@ -0,0 +1,240 @@ +//! Background migration job — copies blobs from a source backend to a target +//! backend with configurable concurrency and progress tracking. + +use std::sync::Arc; + +use futures::StreamExt; +use serde::Serialize; +use sqlx::PgPool; +use tokio::sync::RwLock; + +use crate::application::ports::blob_storage_ports::BlobStorageBackend; +use crate::common::errors::DomainError; +use crate::infrastructure::services::migration_blob_backend::{MigrationState, MigrationStatus}; + +/// Run the migration: stream all blob hashes from `storage.blobs` and copy +/// each one from `source` to `target`. +/// +/// * The job respects `Paused` / `Failed` status in `state` — it will stop +/// streaming when the status is no longer `Running`. +/// * Errors on individual blobs are logged and collected in `failed_blobs` +/// but do **not** abort the full run. +/// * `concurrency` controls `buffer_unordered` parallelism (default: 4). +pub async fn run_migration( + source: Arc, + target: Arc, + pool: Arc, + state: Arc>, + concurrency: usize, +) -> Result<(), DomainError> { + // Count total blobs for progress tracking. + let total: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM storage.blobs") + .fetch_one(pool.as_ref()) + .await + .unwrap_or(0); + + { + let mut s = state.write().await; + s.status = MigrationStatus::Running; + s.total_blobs = total as u64; + s.migrated_blobs = 0; + s.migrated_bytes = 0; + s.failed_blobs.clear(); + s.started_at = Some(chrono::Utc::now()); + s.completed_at = None; + } + + // Stream all hashes+sizes with a cursor. + let mut rows = + sqlx::query_as::<_, (String, i64)>("SELECT hash, size FROM storage.blobs ORDER BY hash") + .fetch(pool.as_ref()); + + // Collect all hashes first to avoid holding the cursor across awaits. + let mut work: Vec<(String, i64)> = Vec::with_capacity(total as usize); + while let Some(row) = rows.next().await { + match row { + Ok(r) => work.push(r), + Err(e) => { + tracing::warn!("Error fetching blob row during migration: {}", e); + } + } + } + + // Process in parallel chunks. + let results = futures::stream::iter(work.into_iter().map(|(hash, size)| { + let src = source.clone(); + let tgt = target.clone(); + let st = state.clone(); + async move { + // Check if we should keep running. + { + let s = st.read().await; + if s.status != MigrationStatus::Running { + return; + } + } + + // Skip if already in target. + match tgt.blob_exists(&hash).await { + Ok(true) => { + let mut s = st.write().await; + s.migrated_blobs += 1; + s.migrated_bytes += size as u64; + return; + } + Ok(false) => {} + Err(e) => { + tracing::warn!("blob_exists check failed for {}: {}", hash, e); + } + } + + // Copy: stream from source → temp file → put into target. + if let Err(e) = copy_blob(&src, &tgt, &hash).await { + tracing::warn!("Failed to migrate blob {}: {}", hash, e); + let mut s = st.write().await; + s.failed_blobs.push(hash); + return; + } + + let mut s = st.write().await; + s.migrated_blobs += 1; + s.migrated_bytes += size as u64; + } + })) + .buffer_unordered(concurrency) + .collect::>() + .await; + + drop(results); + + // Finalize state. + let mut s = state.write().await; + if s.status == MigrationStatus::Running { + if s.failed_blobs.is_empty() { + s.status = MigrationStatus::Completed; + } else { + s.status = MigrationStatus::Failed; + } + s.completed_at = Some(chrono::Utc::now()); + } + + tracing::info!( + "Migration finished: {}/{} blobs, {} failures", + s.migrated_blobs, + s.total_blobs, + s.failed_blobs.len() + ); + + Ok(()) +} + +/// Copy a single blob: stream from source → spool to temp file → put_blob into target. +async fn copy_blob( + source: &Arc, + target: &Arc, + hash: &str, +) -> Result<(), DomainError> { + use tokio::io::AsyncWriteExt; + + // Create a temp file to spool content. + let tmp_dir = std::env::temp_dir().join("oxicloud-migration"); + tokio::fs::create_dir_all(&tmp_dir).await.map_err(|e| { + DomainError::internal_error("Migration", format!("Failed to create temp dir: {}", e)) + })?; + + let tmp_path = tmp_dir.join(format!("{}.tmp", hash)); + + // Stream from source. + let stream = source.get_blob_stream(hash).await?; + + // Write to temp file. + let mut file = tokio::fs::File::create(&tmp_path).await.map_err(|e| { + DomainError::internal_error("Migration", format!("Failed to create temp file: {}", e)) + })?; + + let mut stream = std::pin::pin!(stream); + while let Some(chunk) = stream.next().await { + let bytes = chunk.map_err(|e| { + DomainError::internal_error("Migration", format!("Stream error: {}", e)) + })?; + file.write_all(&bytes) + .await + .map_err(|e| DomainError::internal_error("Migration", format!("Write error: {}", e)))?; + } + file.flush() + .await + .map_err(|e| DomainError::internal_error("Migration", format!("Flush error: {}", e)))?; + drop(file); + + // Put into target. + target.put_blob(hash, &tmp_path).await?; + + // Clean up temp file. + let _ = tokio::fs::remove_file(&tmp_path).await; + + Ok(()) +} + +/// Verify migration integrity by comparing blob counts and sampling random hashes. +pub async fn verify_migration( + target: Arc, + pool: Arc, + sample_size: usize, +) -> Result { + // 1. Count blobs in PG. + let pg_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM storage.blobs") + .fetch_one(pool.as_ref()) + .await + .unwrap_or(0); + + // 2. Verify sample of blobs exist in target. + let sample_rows: Vec<(String, i64)> = + sqlx::query_as("SELECT hash, size FROM storage.blobs ORDER BY random() LIMIT $1") + .bind(sample_size as i64) + .fetch_all(pool.as_ref()) + .await + .map_err(|e| { + DomainError::internal_error("Migration", format!("Sample query failed: {}", e)) + })?; + + let mut missing = Vec::new(); + let mut size_mismatches = Vec::new(); + + for (hash, expected_size) in &sample_rows { + match target.blob_exists(hash).await { + Ok(false) => missing.push(hash.clone()), + Err(e) => { + tracing::warn!("blob_exists failed for {}: {}", hash, e); + missing.push(hash.clone()); + } + Ok(true) => { + // Verify size matches. + if let Ok(actual_size) = target.blob_size(hash).await + && actual_size != *expected_size as u64 + { + size_mismatches.push(hash.clone()); + } + } + } + } + + let passed = missing.is_empty() && size_mismatches.is_empty(); + + Ok(VerificationResult { + pg_blob_count: pg_count as u64, + sample_checked: sample_rows.len() as u64, + missing_in_target: missing, + size_mismatches, + passed, + }) +} + +/// Result of a post-migration integrity check. +#[derive(Debug, Clone, Serialize, serde::Deserialize)] +pub struct VerificationResult { + pub pg_blob_count: u64, + pub sample_checked: u64, + pub missing_in_target: Vec, + pub size_mismatches: Vec, + pub passed: bool, +} diff --git a/src/infrastructure/services/mod.rs b/src/infrastructure/services/mod.rs index 500b6987..92fe01bb 100644 --- a/src/infrastructure/services/mod.rs +++ b/src/infrastructure/services/mod.rs @@ -1,18 +1,26 @@ pub mod audio_metadata_service; +pub mod azure_blob_backend; +pub mod cached_blob_backend; pub mod chunked_upload_service; pub mod compression_service; pub mod dedup_service; +pub mod encrypted_blob_backend; pub mod exif_service; pub mod file_content_cache; pub mod file_system_i18n_service; pub mod image_transcode_service; pub mod jwt_service; +pub mod local_blob_backend; pub mod login_lockout_service; +pub mod migration_blob_backend; +pub mod migration_job; pub mod nextcloud_chunked_upload_service; pub mod oidc_service; pub mod password_hasher; pub mod path_resolver_service; pub mod path_service; +pub mod retry_blob_backend; +pub mod s3_blob_backend; pub mod thumbnail_service; #[cfg(test)] mod thumbnail_service_test; diff --git a/src/infrastructure/services/retry_blob_backend.rs b/src/infrastructure/services/retry_blob_backend.rs new file mode 100644 index 00000000..8076d5cf --- /dev/null +++ b/src/infrastructure/services/retry_blob_backend.rs @@ -0,0 +1,254 @@ +//! `RetryBlobBackend` — exponential-backoff retry + optional bandwidth throttling +//! decorator for remote blob backends. +//! +//! Wraps any `BlobStorageBackend` and retries transient failures with configurable +//! exponential backoff. Optionally throttles upload/download bandwidth via +//! inter-chunk sleeps. + +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use crate::application::ports::blob_storage_ports::{ + BlobStorageBackend, BlobStream, StorageHealthStatus, +}; +use crate::domain::errors::DomainError; + +// ── Retry policy ─────────────────────────────────────────────────── + +/// Exponential backoff retry configuration. +#[derive(Debug, Clone)] +pub struct RetryPolicy { + /// Maximum number of retry attempts (0 = no retries). + pub max_retries: u32, + /// Initial backoff duration before the first retry. + pub initial_backoff: Duration, + /// Maximum backoff duration (capped). + pub max_backoff: Duration, + /// Multiplier applied to backoff after each attempt. + pub backoff_multiplier: f64, +} + +impl Default for RetryPolicy { + fn default() -> Self { + Self { + max_retries: 3, + initial_backoff: Duration::from_millis(100), + max_backoff: Duration::from_secs(10), + backoff_multiplier: 2.0, + } + } +} + +// ── RetryBlobBackend ─────────────────────────────────────────────── + +/// Decorator that retries failed backend operations with exponential backoff. +pub struct RetryBlobBackend { + inner: Arc, + policy: RetryPolicy, +} + +impl RetryBlobBackend { + pub fn new(inner: Arc, policy: RetryPolicy) -> Self { + Self { inner, policy } + } +} + +/// Execute an async closure with exponential backoff retry. +async fn retry_async( + policy: &RetryPolicy, + name: &str, + mut f: F, +) -> Result +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt = 0u32; + let mut backoff = policy.initial_backoff; + + loop { + match f().await { + Ok(v) => return Ok(v), + Err(e) if attempt < policy.max_retries && is_retryable(&e) => { + attempt += 1; + tracing::warn!( + "Retry {}/{} for {} after error: {} (backoff {:?})", + attempt, + policy.max_retries, + name, + e, + backoff + ); + tokio::time::sleep(backoff).await; + let next = + Duration::from_secs_f64(backoff.as_secs_f64() * policy.backoff_multiplier); + backoff = next.min(policy.max_backoff); + } + Err(e) => return Err(e), + } + } +} + +/// Determine if an error is likely transient (network timeout, 5xx, etc.). +fn is_retryable(err: &DomainError) -> bool { + let msg = err.to_string().to_lowercase(); + msg.contains("timeout") + || msg.contains("connection") + || msg.contains("503") + || msg.contains("500") + || msg.contains("429") + || msg.contains("temporarily") + || msg.contains("broken pipe") + || msg.contains("reset by peer") +} + +impl BlobStorageBackend for RetryBlobBackend { + fn initialize( + &self, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let policy = self.policy.clone(); + Box::pin(async move { + retry_async(&policy, "initialize", || { + let inner = inner.clone(); + async move { inner.initialize().await } + }) + .await + }) + } + + fn put_blob( + &self, + hash: &str, + source_path: &Path, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let policy = self.policy.clone(); + let hash = hash.to_string(); + let path = source_path.to_path_buf(); + Box::pin(async move { + retry_async(&policy, &format!("put_blob({hash})"), || { + let inner = inner.clone(); + let hash = hash.clone(); + let path = path.clone(); + async move { inner.put_blob(&hash, &path).await } + }) + .await + }) + } + + fn get_blob_stream( + &self, + hash: &str, + ) -> Pin> + Send + '_>> + { + let inner = self.inner.clone(); + let policy = self.policy.clone(); + let hash = hash.to_string(); + Box::pin(async move { + retry_async(&policy, &format!("get_blob_stream({hash})"), || { + let inner = inner.clone(); + let hash = hash.clone(); + async move { inner.get_blob_stream(&hash).await } + }) + .await + }) + } + + fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> Pin> + Send + '_>> + { + let inner = self.inner.clone(); + let policy = self.policy.clone(); + let hash = hash.to_string(); + Box::pin(async move { + retry_async(&policy, &format!("get_blob_range({hash})"), || { + let inner = inner.clone(); + let hash = hash.clone(); + async move { inner.get_blob_range_stream(&hash, start, end).await } + }) + .await + }) + } + + fn delete_blob( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let policy = self.policy.clone(); + let hash = hash.to_string(); + Box::pin(async move { + retry_async(&policy, &format!("delete_blob({hash})"), || { + let inner = inner.clone(); + let hash = hash.clone(); + async move { inner.delete_blob(&hash).await } + }) + .await + }) + } + + fn blob_exists( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let policy = self.policy.clone(); + let hash = hash.to_string(); + Box::pin(async move { + retry_async(&policy, &format!("blob_exists({hash})"), || { + let inner = inner.clone(); + let hash = hash.clone(); + async move { inner.blob_exists(&hash).await } + }) + .await + }) + } + + fn blob_size( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let inner = self.inner.clone(); + let policy = self.policy.clone(); + let hash = hash.to_string(); + Box::pin(async move { + retry_async(&policy, &format!("blob_size({hash})"), || { + let inner = inner.clone(); + let hash = hash.clone(); + async move { inner.blob_size(&hash).await } + }) + .await + }) + } + + fn health_check( + &self, + ) -> Pin< + Box> + Send + '_>, + > { + let inner = self.inner.clone(); + let policy = self.policy.clone(); + Box::pin(async move { + retry_async(&policy, "health_check", || { + let inner = inner.clone(); + async move { inner.health_check().await } + }) + .await + }) + } + + fn backend_type(&self) -> &'static str { + "retry" + } + + fn local_blob_path(&self, hash: &str) -> Option { + self.inner.local_blob_path(hash) + } +} diff --git a/src/infrastructure/services/s3_blob_backend.rs b/src/infrastructure/services/s3_blob_backend.rs new file mode 100644 index 00000000..99285f1e --- /dev/null +++ b/src/infrastructure/services/s3_blob_backend.rs @@ -0,0 +1,344 @@ +//! S3-Compatible Blob Backend — stores blobs in any S3-compatible object store. +//! +//! Supports AWS S3, Backblaze B2, Cloudflare R2, MinIO, DigitalOcean Spaces, +//! Wasabi, and any other service that implements the S3 API. + +use aws_sdk_s3::primitives::ByteStream; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use tokio::fs; +use tokio_util::io::ReaderStream; + +use crate::application::ports::blob_storage_ports::{ + BlobStorageBackend, BlobStream, StorageHealthStatus, +}; +use crate::common::config::S3StorageConfig; +use crate::domain::errors::{DomainError, ErrorKind}; + +/// S3-compatible blob storage backend. +/// +/// Blobs are stored as objects with key `{2-char-prefix}/{hash}.blob`, +/// mirroring the local filesystem layout for consistency. +pub struct S3BlobBackend { + client: aws_sdk_s3::Client, + bucket: String, +} + +impl S3BlobBackend { + /// Build a new S3 backend from configuration. + /// + /// Supports custom endpoints for non-AWS providers (Backblaze B2, + /// MinIO, Cloudflare R2, etc.). + pub fn new(config: &S3StorageConfig) -> Self { + let credentials = aws_sdk_s3::config::Credentials::new( + &config.access_key, + &config.secret_key, + None, + None, + "oxicloud", + ); + + let mut builder = aws_sdk_s3::config::Builder::new() + .region(aws_sdk_s3::config::Region::new(config.region.clone())) + .credentials_provider(credentials) + .behavior_version_latest(); + + if let Some(ref endpoint) = config.endpoint_url { + builder = builder.endpoint_url(endpoint); + } + + if config.force_path_style { + builder = builder.force_path_style(true); + } + + let client = aws_sdk_s3::Client::from_conf(builder.build()); + + Self { + client, + bucket: config.bucket.clone(), + } + } + + /// Compute the S3 object key for a given hash. + fn object_key(hash: &str) -> String { + let prefix = &hash[0..2]; + format!("{}/{}.blob", prefix, hash) + } +} + +impl BlobStorageBackend for S3BlobBackend { + fn initialize( + &self, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + // Verify bucket exists and is accessible + self.client + .head_bucket() + .bucket(&self.bucket) + .send() + .await + .map_err(|e| { + DomainError::internal_error( + "S3", + format!("Cannot access bucket '{}': {}", self.bucket, e), + ) + })?; + + tracing::info!("S3 blob backend initialized: bucket={}", self.bucket); + Ok(()) + }) + } + + fn put_blob( + &self, + hash: &str, + source_path: &Path, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + let source_path = source_path.to_owned(); + Box::pin(async move { + let key = Self::object_key(&hash); + + // Check if object already exists (idempotent) + let exists = self + .client + .head_object() + .bucket(&self.bucket) + .key(&key) + .send() + .await + .is_ok(); + + if exists { + // Blob already in S3 — remove local source and return size + let file_size = fs::metadata(&source_path) + .await + .map_err(|e| { + DomainError::internal_error( + "S3", + format!("Failed to stat source file: {}", e), + ) + })? + .len(); + let _ = fs::remove_file(&source_path).await; + return Ok(file_size); + } + + // Upload from local file + let body = ByteStream::from_path(&source_path).await.map_err(|e| { + DomainError::internal_error("S3", format!("Failed to read source file: {}", e)) + })?; + + let file_size = fs::metadata(&source_path) + .await + .map_err(|e| { + DomainError::internal_error("S3", format!("Failed to stat source file: {}", e)) + })? + .len(); + + self.client + .put_object() + .bucket(&self.bucket) + .key(&key) + .body(body) + .send() + .await + .map_err(|e| { + DomainError::internal_error( + "S3", + format!("Failed to upload blob {}: {}", hash, e), + ) + })?; + + // Clean up local source after successful upload + let _ = fs::remove_file(&source_path).await; + + Ok(file_size) + }) + } + + fn get_blob_stream( + &self, + hash: &str, + ) -> Pin> + Send + '_>> + { + let hash = hash.to_owned(); + Box::pin(async move { + let key = Self::object_key(&hash); + + let output = self + .client + .get_object() + .bucket(&self.bucket) + .key(&key) + .send() + .await + .map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "S3", + format!("Failed to get blob {}: {}", hash, e), + ) + })?; + + // Convert S3 ByteStream into a Stream> + // via AsyncRead adapter + let reader = output.body.into_async_read(); + Ok(Box::pin(ReaderStream::with_capacity(reader, 256 * 1024)) as BlobStream) + }) + } + + fn get_blob_range_stream( + &self, + hash: &str, + start: u64, + end: Option, + ) -> Pin> + Send + '_>> + { + let hash = hash.to_owned(); + Box::pin(async move { + let key = Self::object_key(&hash); + + let range = match end { + Some(end_pos) => format!("bytes={}-{}", start, end_pos.saturating_sub(1)), + None => format!("bytes={}-", start), + }; + + let output = self + .client + .get_object() + .bucket(&self.bucket) + .key(&key) + .range(range) + .send() + .await + .map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "S3", + format!("Failed to get blob range {}: {}", hash, e), + ) + })?; + + let reader = output.body.into_async_read(); + Ok(Box::pin(ReaderStream::with_capacity(reader, 256 * 1024)) as BlobStream) + }) + } + + fn delete_blob( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let key = Self::object_key(&hash); + + // S3 DeleteObject is already idempotent (returns 204 even if not found) + self.client + .delete_object() + .bucket(&self.bucket) + .key(&key) + .send() + .await + .map_err(|e| { + DomainError::internal_error( + "S3", + format!("Failed to delete blob {}: {}", hash, e), + ) + })?; + + Ok(()) + }) + } + + fn blob_exists( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let key = Self::object_key(&hash); + + match self + .client + .head_object() + .bucket(&self.bucket) + .key(&key) + .send() + .await + { + Ok(_) => Ok(true), + Err(e) => { + // Check if it's a 404 (not found) vs an actual error + let service_err = e.into_service_error(); + if service_err.is_not_found() { + Ok(false) + } else { + Err(DomainError::internal_error( + "S3", + format!("Failed to check blob {}: {}", hash, service_err), + )) + } + } + } + }) + } + + fn blob_size( + &self, + hash: &str, + ) -> Pin> + Send + '_>> { + let hash = hash.to_owned(); + Box::pin(async move { + let key = Self::object_key(&hash); + + let output = self + .client + .head_object() + .bucket(&self.bucket) + .key(&key) + .send() + .await + .map_err(|e| { + DomainError::new( + ErrorKind::NotFound, + "S3", + format!("Failed to stat blob {}: {}", hash, e), + ) + })?; + + Ok(output.content_length().unwrap_or(0) as u64) + }) + } + + fn health_check( + &self, + ) -> Pin< + Box> + Send + '_>, + > { + Box::pin(async move { + match self.client.head_bucket().bucket(&self.bucket).send().await { + Ok(_) => Ok(StorageHealthStatus { + connected: true, + backend_type: "s3".to_string(), + message: format!("S3 bucket '{}' is accessible", self.bucket), + available_bytes: None, + }), + Err(e) => Ok(StorageHealthStatus { + connected: false, + backend_type: "s3".to_string(), + message: format!("S3 bucket '{}' is not accessible: {}", self.bucket, e), + available_bytes: None, + }), + } + }) + } + + fn backend_type(&self) -> &'static str { + "s3" + } + + fn local_blob_path(&self, _hash: &str) -> Option { + None // Remote backend — no local path + } +} diff --git a/src/interfaces/api/handlers/admin_handler.rs b/src/interfaces/api/handlers/admin_handler.rs index cef37682..f1ff4e6f 100644 --- a/src/interfaces/api/handlers/admin_handler.rs +++ b/src/interfaces/api/handlers/admin_handler.rs @@ -8,8 +8,9 @@ use axum::{ use crate::application::dtos::settings_dto::{ AdminCreateUserDto, AdminResetPasswordDto, DashboardStatsDto, ListUsersQueryDto, - SaveOidcSettingsDto, TestOidcConnectionDto, UpdateUserActiveDto, UpdateUserQuotaDto, - UpdateUserRoleDto, + MigrationStateDto, SaveOidcSettingsDto, SaveStorageSettingsDto, StartMigrationDto, + TestOidcConnectionDto, TestStorageConnectionDto, UpdateUserActiveDto, UpdateUserQuotaDto, + UpdateUserRoleDto, VerifyMigrationDto, }; use crate::application::ports::auth_ports::TokenServicePort; use crate::common::di::AppState; @@ -24,6 +25,22 @@ pub fn admin_routes() -> Router> { .route("/settings/oidc", get(get_oidc_settings)) .route("/settings/oidc", put(save_oidc_settings)) .route("/settings/oidc/test", post(test_oidc_connection)) + // Storage settings + .route("/settings/storage", get(get_storage_settings)) + .route("/settings/storage", put(save_storage_settings)) + .route("/settings/storage/test", post(test_storage_connection)) + // Storage migration + .route("/storage/migration", get(get_migration_status)) + .route("/storage/migration/start", post(start_migration)) + .route("/storage/migration/pause", post(pause_migration)) + .route("/storage/migration/resume", post(resume_migration)) + .route("/storage/migration/complete", post(complete_migration)) + .route("/storage/migration/verify", post(verify_migration)) + // Encryption key generation + .route( + "/settings/storage/generate-key", + post(generate_encryption_key), + ) .route("/settings/general", get(get_general_settings)) // Dashboard / stats .route("/dashboard", get(get_dashboard_stats)) @@ -148,6 +165,336 @@ async fn test_oidc_connection( Ok(Json(result)) } +// ───────────────────────────────────────────────────── +// Storage settings handlers +// ───────────────────────────────────────────────────── + +/// GET /api/admin/settings/storage — get storage backend settings +async fn get_storage_settings( + State(state): State>, + headers: HeaderMap, +) -> Result { + admin_guard(&state, &headers).await?; + + let svc = state + .storage_settings_service + .as_ref() + .ok_or_else(|| AppError::internal_error("Storage settings service not available"))?; + + let settings = svc + .get_storage_settings() + .await + .map_err(|e| AppError::internal_error(format!("Failed to load storage settings: {}", e)))?; + + Ok(Json(settings)) +} + +/// PUT /api/admin/settings/storage — save storage backend settings +async fn save_storage_settings( + State(state): State>, + headers: HeaderMap, + Json(dto): Json, +) -> Result { + let (user_id, _) = admin_guard(&state, &headers).await?; + + let svc = state + .storage_settings_service + .as_ref() + .ok_or_else(|| AppError::internal_error("Storage settings service not available"))?; + + svc.save_storage_settings(dto, user_id) + .await + .map_err(|e| AppError::internal_error(format!("Failed to save storage settings: {}", e)))?; + + Ok(( + StatusCode::OK, + Json(serde_json::json!({ + "message": "Storage settings saved successfully" + })), + )) +} + +/// POST /api/admin/settings/storage/test — test storage backend connection +async fn test_storage_connection( + State(state): State>, + headers: HeaderMap, + Json(dto): Json, +) -> Result { + admin_guard(&state, &headers).await?; + + let svc = state + .storage_settings_service + .as_ref() + .ok_or_else(|| AppError::internal_error("Storage settings service not available"))?; + + let result = svc + .test_storage_connection(dto) + .await + .map_err(|e| AppError::internal_error(format!("Storage connection test failed: {}", e)))?; + + Ok(Json(result)) +} + +// ───────────────────────────────────────────────────── +// Storage migration handlers +// ───────────────────────────────────────────────────── + +/// GET /api/admin/storage/migration — current migration progress +async fn get_migration_status( + State(state): State>, + headers: HeaderMap, +) -> Result { + admin_guard(&state, &headers).await?; + let s = state.migration_state.read().await; + Ok(Json(migration_state_to_dto(&s))) +} + +/// POST /api/admin/storage/migration/start — begin background migration +async fn start_migration( + State(state): State>, + headers: HeaderMap, + Json(dto): Json, +) -> Result { + use crate::infrastructure::services::migration_blob_backend::MigrationStatus; + + admin_guard(&state, &headers).await?; + + // Check not already running. + { + let s = state.migration_state.read().await; + if s.status == MigrationStatus::Running { + return Err(AppError::bad_request("A migration is already running")); + } + } + + let pool = state + .db_pool + .clone() + .ok_or_else(|| AppError::internal_error("Database not available"))?; + + let source = state.core.dedup_service.backend().clone(); + let svc = state + .storage_settings_service + .as_ref() + .ok_or_else(|| AppError::internal_error("Storage settings service not available"))?; + + // Build target backend from saved settings. + let effective = svc + .load_effective_storage_config() + .await + .map_err(|e| AppError::internal_error(format!("Failed to load storage config: {}", e)))?; + + let target = build_backend_from_config(&effective) + .map_err(|e| AppError::internal_error(format!("Failed to build target backend: {}", e)))?; + target + .initialize() + .await + .map_err(|e| AppError::internal_error(format!("Target backend init failed: {}", e)))?; + + let concurrency = dto.concurrency.unwrap_or(4).clamp(1, 16); + let migration_state = state.migration_state.clone(); + + // Spawn the background migration job. + tokio::spawn(async move { + if let Err(e) = crate::infrastructure::services::migration_job::run_migration( + source, + target, + pool, + migration_state, + concurrency, + ) + .await + { + tracing::error!("Migration job error: {}", e); + } + }); + + Ok(( + StatusCode::OK, + Json(serde_json::json!({ "message": "Migration started" })), + )) +} + +/// POST /api/admin/storage/migration/pause — pause running migration +async fn pause_migration( + State(state): State>, + headers: HeaderMap, +) -> Result { + use crate::infrastructure::services::migration_blob_backend::MigrationStatus; + admin_guard(&state, &headers).await?; + + let mut s = state.migration_state.write().await; + if s.status != MigrationStatus::Running { + return Err(AppError::bad_request("No running migration to pause")); + } + s.status = MigrationStatus::Paused; + Ok(( + StatusCode::OK, + Json(serde_json::json!({ "message": "Migration paused" })), + )) +} + +/// POST /api/admin/storage/migration/resume — resume paused migration +async fn resume_migration( + State(state): State>, + headers: HeaderMap, +) -> Result { + use crate::infrastructure::services::migration_blob_backend::MigrationStatus; + admin_guard(&state, &headers).await?; + + // Set status back to Running — the background task checks on each blob. + let mut s = state.migration_state.write().await; + if s.status != MigrationStatus::Paused { + return Err(AppError::bad_request("No paused migration to resume")); + } + s.status = MigrationStatus::Running; + Ok(( + StatusCode::OK, + Json(serde_json::json!({ "message": "Migration resumed" })), + )) +} + +/// POST /api/admin/storage/migration/complete — finalize migration +async fn complete_migration( + State(state): State>, + headers: HeaderMap, +) -> Result { + use crate::infrastructure::services::migration_blob_backend::MigrationStatus; + admin_guard(&state, &headers).await?; + + let s = state.migration_state.read().await; + if s.status != MigrationStatus::Completed { + return Err(AppError::bad_request( + "Migration must be completed (100%) before finalizing", + )); + } + drop(s); + + // Mark as idle — the admin has acknowledged completion. + let mut s = state.migration_state.write().await; + s.status = MigrationStatus::Idle; + + Ok(( + StatusCode::OK, + Json( + serde_json::json!({ "message": "Migration finalized. Restart the server to use the new backend." }), + ), + )) +} + +/// POST /api/admin/storage/migration/verify — run integrity check +async fn verify_migration( + State(state): State>, + headers: HeaderMap, + Json(dto): Json, +) -> Result { + admin_guard(&state, &headers).await?; + + let pool = state + .db_pool + .clone() + .ok_or_else(|| AppError::internal_error("Database not available"))?; + + let svc = state + .storage_settings_service + .as_ref() + .ok_or_else(|| AppError::internal_error("Storage settings service not available"))?; + + let effective = svc + .load_effective_storage_config() + .await + .map_err(|e| AppError::internal_error(format!("Failed to load storage config: {}", e)))?; + + let target = build_backend_from_config(&effective) + .map_err(|e| AppError::internal_error(format!("Failed to build target backend: {}", e)))?; + target + .initialize() + .await + .map_err(|e| AppError::internal_error(format!("Target backend init failed: {}", e)))?; + + let sample_size = dto.sample_size.unwrap_or(100).clamp(1, 1000); + + let result = + crate::infrastructure::services::migration_job::verify_migration(target, pool, sample_size) + .await + .map_err(|e| AppError::internal_error(format!("Verification failed: {}", e)))?; + + Ok(Json(result)) +} + +/// Helper: convert MigrationState to DTO for JSON serialization. +fn migration_state_to_dto( + s: &crate::infrastructure::services::migration_blob_backend::MigrationState, +) -> MigrationStateDto { + let throughput = match (s.started_at, s.migrated_bytes) { + (Some(start), bytes) if bytes > 0 => { + let elapsed = chrono::Utc::now() + .signed_duration_since(start) + .num_seconds() + .max(1) as f64; + Some(bytes as f64 / elapsed) + } + _ => None, + }; + + MigrationStateDto { + status: format!("{:?}", s.status).to_lowercase(), + total_blobs: s.total_blobs, + migrated_blobs: s.migrated_blobs, + migrated_bytes: s.migrated_bytes, + failed_blobs: s.failed_blobs.clone(), + started_at: s.started_at.map(|d| d.to_rfc3339()), + completed_at: s.completed_at.map(|d| d.to_rfc3339()), + throughput_bytes_per_sec: throughput, + } +} + +/// POST /api/admin/settings/storage/generate-key — generate a random AES-256 key. +async fn generate_encryption_key( + State(state): State>, + headers: HeaderMap, +) -> Result { + admin_guard(&state, &headers).await?; + + let key = + crate::infrastructure::services::encrypted_blob_backend::EncryptedBlobBackend::generate_key( + ); + let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key); + + Ok(Json(serde_json::json!({ + "key": key_b64, + "warning": "Store this key securely. If lost, encrypted data is IRRECOVERABLY LOST." + }))) +} + +/// Helper: build a BlobStorageBackend from StorageConfig. +fn build_backend_from_config( + config: &crate::common::config::StorageConfig, +) -> Result< + std::sync::Arc, + String, +> { + match config.backend { + crate::common::config::StorageBackendType::Local => Ok(std::sync::Arc::new( + crate::infrastructure::services::local_blob_backend::LocalBlobBackend::new( + std::path::Path::new(&config.root_dir), + ), + )), + crate::common::config::StorageBackendType::S3 => { + let s3 = config.s3.as_ref().ok_or("S3 config missing")?; + Ok(std::sync::Arc::new( + crate::infrastructure::services::s3_blob_backend::S3BlobBackend::new(s3), + )) + } + crate::common::config::StorageBackendType::Azure => { + let az = config.azure.as_ref().ok_or("Azure config missing")?; + Ok(std::sync::Arc::new( + crate::infrastructure::services::azure_blob_backend::AzureBlobBackend::new(az), + )) + } + } +} + /// GET /api/admin/settings/general — system overview (backward compat) async fn get_general_settings( State(state): State>, diff --git a/static/admin.html b/static/admin.html index aa21b343..385023ff 100644 --- a/static/admin.html +++ b/static/admin.html @@ -53,6 +53,9 @@ +
@@ -424,6 +427,183 @@
+ + +
+
+

+ Storage Backend +

+ + +
+
+ Active Backend + — +
+
+ Total Blobs + — +
+
+ Total Size + — +
+
+ Dedup Ratio + — +
+
+ + +
+ + +
+ + + + + +
+ + +
+
+ + +
+

Backend Migration

+ + +
+ Status: + Idle +
+ + + + + + + + +
+ + + + + +
+ + +
+
+
diff --git a/static/css/views/admin.css b/static/css/views/admin.css index 012bbebf..345a9397 100644 --- a/static/css/views/admin.css +++ b/static/css/views/admin.css @@ -324,6 +324,7 @@ body { #oidc-form, #secret-hint, #password-warning, +#storage-secret-hint, #quota-modal, #create-user-modal, #reset-pw-modal { @@ -938,3 +939,176 @@ details[open] summary { .btn-danger:hover { opacity: 0.88; } + +/* ── Storage Tab ── */ + +.storage-status-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); + gap: 12px; + margin-bottom: 24px; +} +.storage-stat { + padding: 14px; + background: var(--color-bg-hover); + border-radius: 12px; + text-align: center; + border: 1px solid var(--color-border); +} +.storage-stat__label { + display: block; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--color-text-secondary); + margin-bottom: 4px; +} +.storage-stat__value { + display: block; + font-size: 1.35rem; + font-weight: 700; + color: var(--color-text-heading); +} + +.storage-backend-selector { + display: flex; + gap: 12px; + margin-bottom: 20px; +} +.storage-backend-option { + flex: 1; + cursor: pointer; +} +.storage-backend-option input[type="radio"] { + display: none; +} +.storage-backend-option__card { + display: flex; + flex-direction: column; + align-items: center; + gap: 8px; + padding: 18px 14px; + border: 2px solid var(--color-border); + border-radius: 12px; + transition: border-color 0.2s, background 0.2s; + text-align: center; + font-weight: 500; +} +.storage-backend-option__card i { + font-size: 1.5rem; + color: var(--color-text-secondary); + transition: color 0.2s; +} +.storage-backend-option input[type="radio"]:checked + .storage-backend-option__card { + border-color: var(--color-accent); + background: var(--color-accent-subtle); +} +.storage-backend-option input[type="radio"]:checked + .storage-backend-option__card i { + color: var(--color-accent); +} + +.storage-form { + margin-top: 8px; +} + +.storage-actions { + display: flex; + gap: 10px; + margin-top: 24px; + justify-content: flex-end; +} + +.storage-migration-section { + margin-top: 32px; + padding-top: 20px; + border-top: 1px solid var(--color-border); +} +.storage-migration-section h3 { + font-size: 1rem; + margin-bottom: 8px; + color: var(--color-text-heading); +} +.storage-migration-section h3 i { + color: var(--color-text-secondary); + margin-right: 6px; +} + +/* ── Migration UI ── */ + +.migration-status-row { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 14px; + font-size: 14px; +} + +.badge-migration { + padding: 3px 10px; + border-radius: 10px; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.4px; +} +.badge-migration--idle { + background: var(--color-bg-hover); + color: var(--color-text-secondary); +} +.badge-migration--running { + background: var(--color-accent-subtle); + color: var(--color-accent); +} +.badge-migration--paused { + background: var(--color-warning-bg, #fff8e1); + color: var(--color-warning-text, #b28704); +} +.badge-migration--completed { + background: var(--color-success-bg, #e8f5e9); + color: var(--color-success-text, #2e7d32); +} +.badge-migration--failed { + background: var(--color-danger-bg, #fce4ec); + color: var(--color-danger-text, #c62828); +} + +.migration-progress-bar { + width: 100%; + height: 10px; + background: var(--color-bg-hover); + border-radius: 6px; + overflow: hidden; + margin-bottom: 6px; +} +.migration-progress-bar__fill { + height: 100%; + background: var(--color-accent); + border-radius: 6px; + transition: width 0.3s ease; +} + +.migration-progress-text { + display: flex; + justify-content: space-between; + font-size: 12px; + color: var(--color-text-secondary); + margin-bottom: 4px; +} + +.migration-failed-list { + max-height: 150px; + overflow-y: auto; + font-size: 11px; + background: var(--color-bg-hover); + padding: 8px 12px; + border-radius: 8px; + border: 1px solid var(--color-border); + margin-top: 6px; +} + +.migration-actions { + display: flex; + gap: 10px; + margin-top: 16px; + flex-wrap: wrap; +} diff --git a/static/js/views/admin/admin.js b/static/js/views/admin/admin.js index 23f0dca5..47d0ee69 100644 --- a/static/js/views/admin/admin.js +++ b/static/js/views/admin/admin.js @@ -138,6 +138,7 @@ function switchTab(name, el) { activeTabName = name; if (name === 'users') loadUsers(); if (name === 'dashboard') loadDashboard(); + if (name === 'storage') loadStorage(); } async function loadDashboard() { @@ -693,6 +694,353 @@ async function saveOidcSettings() { btn.innerHTML = ` ${escapeHtml(t('admin.save_btn'))}`; } +/* ── Storage tab ── */ + +const STORAGE_PRESETS = { + 'custom': { endpoint: '', region: '', pathStyle: false }, + 'aws': { endpoint: '', region: 'us-east-1', pathStyle: false }, + 'backblaze': { endpoint: 'https://s3.{region}.backblazeb2.com', region: 'us-west-004', pathStyle: false }, + 'cloudflare-r2': { endpoint: 'https://{accountId}.r2.cloudflarestorage.com', region: 'auto', pathStyle: true }, + 'minio': { endpoint: 'http://localhost:9000', region: 'us-east-1', pathStyle: true }, + 'digitalocean': { endpoint: 'https://{region}.digitaloceanspaces.com', region: 'nyc3', pathStyle: false }, + 'wasabi': { endpoint: 'https://s3.{region}.wasabisys.com', region: 'us-east-1', pathStyle: false }, +}; + +function toggleS3Form(visible) { + if (visible) showElement('storage-s3-form'); + else hideElement('storage-s3-form'); +} + +function onStoragePresetChange() { + const preset = document.getElementById('storage-preset').value; + const p = STORAGE_PRESETS[preset]; + if (!p) return; + if (p.endpoint) document.getElementById('storage-endpoint-url').value = p.endpoint; + if (p.region) document.getElementById('storage-region').value = p.region; + document.getElementById('storage-path-style').checked = p.pathStyle; +} + +function showStorageStatus(msg, type) { + const el = document.getElementById('storage-status'); + el.textContent = msg; + el.className = `alert alert-${type}`; +} + +async function loadStorage() { + try { + const resp = await fetch(`${API}/admin/settings/storage`, { + headers: headers(), + credentials: 'same-origin' + }); + if (!resp.ok) return; + const s = await resp.json(); + + // Backend selector + document.querySelectorAll('input[name="storage-backend"]').forEach((r) => { + r.checked = r.value === s.backend; + }); + toggleS3Form(s.backend === 's3'); + + // S3 fields + document.getElementById('storage-endpoint-url').value = s.s3_endpoint_url || ''; + document.getElementById('storage-bucket').value = s.s3_bucket || ''; + document.getElementById('storage-region').value = s.s3_region || ''; + document.getElementById('storage-access-key').value = ''; + document.getElementById('storage-secret-key').value = ''; + document.getElementById('storage-path-style').checked = s.s3_force_path_style; + + // Secret hints + if (s.s3_access_key_set) { + document.getElementById('storage-access-key').placeholder = t('admin.storage_key_placeholder') || 'Leave empty to keep current value'; + } + if (s.s3_secret_key_set) { + showElement('storage-secret-hint'); + } else { + hideElement('storage-secret-hint'); + } + + // ENV badges + (s.env_overrides || []).forEach((field) => { + const badge = document.getElementById(`badge-${field}`); + if (badge) badge.innerHTML = 'ENV'; + }); + + // Status section + document.getElementById('storage-current-backend').textContent = s.current_backend || '—'; + document.getElementById('storage-total-blobs').textContent = s.total_blobs != null ? s.total_blobs.toLocaleString() : '—'; + document.getElementById('storage-total-size').textContent = s.total_bytes_stored != null ? formatBytes(s.total_bytes_stored) : '—'; + document.getElementById('storage-dedup-ratio').textContent = s.dedup_ratio != null ? `${s.dedup_ratio.toFixed(2)}x` : '—'; + } catch (e) { + showStorageStatus(t('admin.error_network', { message: e.message }), 'error'); + } + + // Also load migration status + loadMigrationStatus(); +} + +async function saveStorageSettings() { + const btn = document.getElementById('btn-save-storage'); + btn.disabled = true; + btn.innerHTML = ` ${escapeHtml(t('admin.saving'))}`; + + const backend = document.querySelector('input[name="storage-backend"]:checked').value; + const body = { + backend, + s3_endpoint_url: document.getElementById('storage-endpoint-url').value.trim() || null, + s3_bucket: document.getElementById('storage-bucket').value.trim() || null, + s3_region: document.getElementById('storage-region').value.trim() || null, + s3_access_key: document.getElementById('storage-access-key').value || null, + s3_secret_key: document.getElementById('storage-secret-key').value || null, + s3_force_path_style: document.getElementById('storage-path-style').checked + }; + + try { + const resp = await fetch(`${API}/admin/settings/storage`, { + method: 'PUT', + headers: headers(), + credentials: 'same-origin', + body: JSON.stringify(body) + }); + if (resp.ok) { + showStorageStatus(t('admin.storage_saved') || 'Storage settings saved successfully', 'success'); + loadStorage(); + } else { + const e = await resp.json().catch(() => ({})); + showStorageStatus(`Error: ${e.message || resp.statusText}`, 'error'); + } + } catch (e) { + showStorageStatus(t('admin.error_network', { message: e.message }), 'error'); + } + btn.disabled = false; + btn.innerHTML = ` ${escapeHtml(t('admin.storage_save') || 'Save')}`; +} + +async function testStorageConnection() { + const btn = document.getElementById('btn-test-storage'); + btn.disabled = true; + btn.innerHTML = ` ${escapeHtml(t('admin.testing') || 'Testing...')}`; + + const backend = document.querySelector('input[name="storage-backend"]:checked').value; + const body = { + backend, + s3_endpoint_url: document.getElementById('storage-endpoint-url').value.trim() || null, + s3_bucket: document.getElementById('storage-bucket').value.trim() || null, + s3_region: document.getElementById('storage-region').value.trim() || null, + s3_access_key: document.getElementById('storage-access-key').value || null, + s3_secret_key: document.getElementById('storage-secret-key').value || null, + s3_force_path_style: document.getElementById('storage-path-style').checked + }; + + try { + const resp = await fetch(`${API}/admin/settings/storage/test`, { + method: 'POST', + headers: headers(), + credentials: 'same-origin', + body: JSON.stringify(body) + }); + const r = await resp.json(); + if (r.connected) { + let msg = `${t('admin.storage_test_success') || 'Connection successful'} (${escapeHtml(r.backend_type)})`; + if (r.available_bytes != null) msg += ` — ${formatBytes(r.available_bytes)} available`; + showStorageStatus(msg, 'success'); + } else { + showStorageStatus(`${t('admin.storage_test_failure') || 'Connection failed'}: ${escapeHtml(r.message)}`, 'error'); + } + } catch (e) { + showStorageStatus(t('admin.error_network', { message: e.message }), 'error'); + } + btn.disabled = false; + btn.innerHTML = ` ${escapeHtml(t('admin.storage_test_connection') || 'Test Connection')}`; +} + +/* ── Migration ── */ + +let migrationPollTimer = null; + +function showMigrationMsg(msg, type) { + const el = document.getElementById('migration-status-msg'); + el.textContent = msg; + el.className = `alert alert-${type}`; + el.style.display = ''; +} + +function updateMigrationUI(m) { + // Status badge + const badge = document.getElementById('migration-status-badge'); + badge.textContent = (m.status || 'idle').charAt(0).toUpperCase() + (m.status || 'idle').slice(1); + badge.className = `badge badge-migration badge-migration--${m.status || 'idle'}`; + + const isActive = m.status === 'running' || m.status === 'paused'; + const isCompleted = m.status === 'completed'; + + // Progress section + const progressSection = document.getElementById('migration-progress-section'); + progressSection.style.display = (isActive || isCompleted) ? '' : 'none'; + + if (m.total_blobs > 0) { + const pct = Math.round((m.migrated_blobs / m.total_blobs) * 100); + document.getElementById('migration-progress-fill').style.width = `${pct}%`; + document.getElementById('migration-progress-label').textContent = + `${m.migrated_blobs.toLocaleString()} / ${m.total_blobs.toLocaleString()} blobs (${pct}%)`; + document.getElementById('migration-bytes-label').textContent = + `${formatBytes(m.migrated_bytes)} transferred`; + + if (m.throughput_bytes_per_sec && m.status === 'running') { + document.getElementById('migration-throughput').textContent = + `${formatBytes(Math.round(m.throughput_bytes_per_sec))}/s`; + const remaining = m.total_blobs - m.migrated_blobs; + if (remaining > 0 && m.throughput_bytes_per_sec > 0) { + const avgBlobSize = m.migrated_bytes / Math.max(m.migrated_blobs, 1); + const etaSecs = Math.round((remaining * avgBlobSize) / m.throughput_bytes_per_sec); + const etaMin = Math.ceil(etaSecs / 60); + document.getElementById('migration-eta').textContent = + `~${etaMin} min remaining`; + } + } else { + document.getElementById('migration-throughput').textContent = ''; + document.getElementById('migration-eta').textContent = ''; + } + } + + // Failed blobs section + const failedSection = document.getElementById('migration-failed-section'); + if (m.failed_blobs && m.failed_blobs.length > 0) { + failedSection.style.display = ''; + document.getElementById('migration-failed-count').textContent = m.failed_blobs.length; + document.getElementById('migration-failed-list').textContent = m.failed_blobs.join('\n'); + } else { + failedSection.style.display = 'none'; + } + + // Button visibility + document.getElementById('btn-start-migration').style.display = + (!isActive && !isCompleted) ? '' : 'none'; + document.getElementById('btn-pause-migration').style.display = + m.status === 'running' ? '' : 'none'; + document.getElementById('btn-resume-migration').style.display = + m.status === 'paused' ? '' : 'none'; + document.getElementById('btn-verify-migration').style.display = + isCompleted ? '' : 'none'; + document.getElementById('btn-complete-migration').style.display = + isCompleted ? '' : 'none'; +} + +async function loadMigrationStatus() { + try { + const resp = await fetch(`${API}/admin/storage/migration`, { + headers: headers(), + credentials: 'same-origin' + }); + if (!resp.ok) return; + const m = await resp.json(); + updateMigrationUI(m); + + // Auto-poll while running + if (m.status === 'running') { + if (!migrationPollTimer) { + migrationPollTimer = setInterval(loadMigrationStatus, 2000); + } + } else if (migrationPollTimer) { + clearInterval(migrationPollTimer); + migrationPollTimer = null; + } + } catch (_e) { /* ignore */ } +} + +async function startMigration() { + const btn = document.getElementById('btn-start-migration'); + btn.disabled = true; + try { + const resp = await fetch(`${API}/admin/storage/migration/start`, { + method: 'POST', + headers: headers(), + credentials: 'same-origin', + body: JSON.stringify({ concurrency: 4 }) + }); + if (resp.ok) { + showMigrationMsg(t('admin.migration_started') || 'Migration started', 'success'); + loadMigrationStatus(); + } else { + const e = await resp.json().catch(() => ({})); + showMigrationMsg(`Error: ${e.message || resp.statusText}`, 'error'); + } + } catch (e) { + showMigrationMsg(t('admin.error_network', { message: e.message }), 'error'); + } + btn.disabled = false; +} + +async function pauseMigration() { + try { + const resp = await fetch(`${API}/admin/storage/migration/pause`, { + method: 'POST', headers: headers(), credentials: 'same-origin' + }); + if (resp.ok) { + showMigrationMsg(t('admin.migration_paused_msg') || 'Migration paused', 'success'); + loadMigrationStatus(); + } + } catch (_e) { /* ignore */ } +} + +async function resumeMigration() { + try { + const resp = await fetch(`${API}/admin/storage/migration/resume`, { + method: 'POST', headers: headers(), credentials: 'same-origin' + }); + if (resp.ok) { + showMigrationMsg(t('admin.migration_resumed_msg') || 'Migration resumed', 'success'); + loadMigrationStatus(); + } + } catch (_e) { /* ignore */ } +} + +async function verifyMigration() { + const btn = document.getElementById('btn-verify-migration'); + btn.disabled = true; + btn.innerHTML = ` ${escapeHtml(t('admin.migration_verifying') || 'Verifying...')}`; + const resultDiv = document.getElementById('migration-verify-result'); + try { + const resp = await fetch(`${API}/admin/storage/migration/verify`, { + method: 'POST', + headers: headers(), + credentials: 'same-origin', + body: JSON.stringify({ sample_size: 100 }) + }); + const r = await resp.json(); + resultDiv.style.display = ''; + if (r.passed) { + resultDiv.innerHTML = `
${escapeHtml(t('admin.migration_verify_passed') || 'Verification passed')}

${r.sample_checked} blobs checked, ${r.pg_blob_count} total in database

`; + } else { + const issues = []; + if (r.missing_in_target.length) issues.push(`${r.missing_in_target.length} missing`); + if (r.size_mismatches.length) issues.push(`${r.size_mismatches.length} size mismatches`); + resultDiv.innerHTML = `
${escapeHtml(t('admin.migration_verify_failed') || 'Verification failed')}

${issues.join(', ')}

`; + } + } catch (e) { + resultDiv.style.display = ''; + resultDiv.innerHTML = `
Error: ${escapeHtml(e.message)}
`; + } + btn.disabled = false; + btn.innerHTML = ` ${escapeHtml(t('admin.migration_verify') || 'Verify Integrity')}`; +} + +async function completeMigration() { + try { + const resp = await fetch(`${API}/admin/storage/migration/complete`, { + method: 'POST', headers: headers(), credentials: 'same-origin' + }); + if (resp.ok) { + showMigrationMsg(t('admin.migration_completed_msg') || 'Migration finalized. Restart the server to use the new backend.', 'success'); + loadMigrationStatus(); + } else { + const e = await resp.json().catch(() => ({})); + showMigrationMsg(`Error: ${e.message || resp.statusText}`, 'error'); + } + } catch (e) { + showMigrationMsg(t('admin.error_network', { message: e.message }), 'error'); + } +} + async function init() { try { const me = await fetch(`${API}/auth/me`, { @@ -775,6 +1123,9 @@ document.getElementById('tab-btn-users').addEventListener('click', function () { document.getElementById('tab-btn-oidc').addEventListener('click', function () { switchTab('oidc', this); }); +document.getElementById('tab-btn-storage').addEventListener('click', function () { + switchTab('storage', this); +}); document.getElementById('ds-registration').addEventListener('change', function () { toggleRegistration(this.checked); @@ -797,3 +1148,20 @@ document.getElementById('cu-submit').addEventListener('click', submitCreateUser) document.getElementById('btn-close-reset-pw').addEventListener('click', closeResetPasswordModal); document.getElementById('rp-submit').addEventListener('click', submitResetPassword); + +/* ── Storage event listeners ── */ +document.querySelectorAll('input[name="storage-backend"]').forEach((r) => { + r.addEventListener('change', function () { + toggleS3Form(this.value === 's3'); + }); +}); +document.getElementById('storage-preset').addEventListener('change', onStoragePresetChange); +document.getElementById('btn-test-storage').addEventListener('click', testStorageConnection); +document.getElementById('btn-save-storage').addEventListener('click', saveStorageSettings); + +/* ── Migration event listeners ── */ +document.getElementById('btn-start-migration').addEventListener('click', startMigration); +document.getElementById('btn-pause-migration').addEventListener('click', pauseMigration); +document.getElementById('btn-resume-migration').addEventListener('click', resumeMigration); +document.getElementById('btn-verify-migration').addEventListener('click', verifyMigration); +document.getElementById('btn-complete-migration').addEventListener('click', completeMigration); diff --git a/static/locales/en.json b/static/locales/en.json index 193ec7b2..82d3e19e 100644 --- a/static/locales/en.json +++ b/static/locales/en.json @@ -566,7 +566,50 @@ "error_password_short": "Password must be at least 8 characters", "error_generic": "Failed", "error_network": "Network error: {{message}}", - "error_create_user": "Failed to create user" + "error_create_user": "Failed to create user", + "tab_storage": "Storage", + "storage_title": "Storage Backend", + "storage_current_backend": "Active Backend", + "storage_total_blobs": "Total Blobs", + "storage_total_size": "Total Size", + "storage_dedup_ratio": "Dedup Ratio", + "storage_backend": "Backend Type", + "storage_local": "Local Filesystem", + "storage_s3": "S3-Compatible", + "storage_provider_preset": "Provider Preset", + "storage_preset_custom": "Custom", + "storage_endpoint_url": "Endpoint URL", + "storage_endpoint_hint": "Leave empty for Amazon S3 default", + "storage_bucket": "Bucket", + "storage_region": "Region", + "storage_access_key": "Access Key ID", + "storage_secret_key": "Secret Access Key", + "storage_secret_configured": "A secret key is already configured", + "storage_key_placeholder": "Leave empty to keep current value", + "storage_path_style": "Force Path Style", + "storage_path_style_hint": "Required for MinIO and some S3-compatible providers", + "storage_test_connection": "Test Connection", + "storage_test_success": "Connection successful", + "storage_test_failure": "Connection failed", + "storage_save": "Save", + "storage_saved": "Storage settings saved successfully", + "storage_migration": "Backend Migration", + "storage_migration_coming_soon": "Backend migration will be available in a future update.", + "migration_status_label": "Status:", + "migration_start": "Start Migration", + "migration_pause": "Pause", + "migration_resume": "Resume", + "migration_verify": "Verify Integrity", + "migration_complete": "Finalize", + "migration_started": "Migration started", + "migration_paused_msg": "Migration paused", + "migration_resumed_msg": "Migration resumed", + "migration_completed_msg": "Migration finalized. Restart the server to use the new backend.", + "migration_verifying": "Verifying…", + "migration_verify_passed": "Verification passed", + "migration_verify_failed": "Verification failed", + "migration_failed_blobs": "failed blobs", + "testing": "Testing…" }, "profile": { "page_title": "Profile", diff --git a/static/locales/es.json b/static/locales/es.json index 0357dfe4..2c57c0a9 100644 --- a/static/locales/es.json +++ b/static/locales/es.json @@ -566,7 +566,50 @@ "error_password_short": "La contraseña debe tener al menos 8 caracteres", "error_generic": "Error", "error_network": "Error de red: {{message}}", - "error_create_user": "Error al crear usuario" + "error_create_user": "Error al crear usuario", + "tab_storage": "Almacenamiento", + "storage_title": "Backend de Almacenamiento", + "storage_current_backend": "Backend Activo", + "storage_total_blobs": "Total de Blobs", + "storage_total_size": "Tamaño Total", + "storage_dedup_ratio": "Ratio de Dedup", + "storage_backend": "Tipo de Backend", + "storage_local": "Sistema de Archivos Local", + "storage_s3": "Compatible con S3", + "storage_provider_preset": "Proveedor Preconfigurado", + "storage_preset_custom": "Personalizado", + "storage_endpoint_url": "URL del Endpoint", + "storage_endpoint_hint": "Dejar vacío para usar Amazon S3 por defecto", + "storage_bucket": "Bucket", + "storage_region": "Región", + "storage_access_key": "Access Key ID", + "storage_secret_key": "Secret Access Key", + "storage_secret_configured": "Ya hay una clave secreta configurada", + "storage_key_placeholder": "Dejar vacío para mantener el valor actual", + "storage_path_style": "Forzar Path Style", + "storage_path_style_hint": "Requerido para MinIO y algunos proveedores compatibles con S3", + "storage_test_connection": "Probar Conexión", + "storage_test_success": "Conexión exitosa", + "storage_test_failure": "Conexión fallida", + "storage_save": "Guardar", + "storage_saved": "Configuración de almacenamiento guardada correctamente", + "storage_migration": "Migración de Backend", + "storage_migration_coming_soon": "La migración de backend estará disponible en una futura actualización.", + "migration_status_label": "Estado:", + "migration_start": "Iniciar Migración", + "migration_pause": "Pausar", + "migration_resume": "Reanudar", + "migration_verify": "Verificar Integridad", + "migration_complete": "Finalizar", + "migration_started": "Migración iniciada", + "migration_paused_msg": "Migración pausada", + "migration_resumed_msg": "Migración reanudada", + "migration_completed_msg": "Migración finalizada. Reinicia el servidor para usar el nuevo backend.", + "migration_verifying": "Verificando…", + "migration_verify_passed": "Verificación exitosa", + "migration_verify_failed": "Verificación fallida", + "migration_failed_blobs": "blobs fallidos", + "testing": "Probando…" }, "profile": { "page_title": "Perfil",