diff --git a/Cargo.lock b/Cargo.lock index b3d44d27..2ee0cc0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,7 +160,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -171,7 +171,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2328,6 +2328,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8874945f036109c72242964c1174cf99434e30cfa45bf45fedc983f50046f8" +dependencies = [ + "hashbag", + "left-right", + "smallvec", +] + [[package]] name = "extism" version = "1.30.0" @@ -2743,6 +2754,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2918,6 +2944,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbag" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7040a10f52cba493ddb09926e15d10a9d8a28043708a405931fe4c6f19fac064" + [[package]] name = "hashbrown" version = "0.12.3" @@ -3441,7 +3473,7 @@ checksum = "af1955a75fa080c677d3972822ec4bad316169ab1cfc6c257a942c2265dbe5fe" dependencies = [ "bitmaps", "rand_core 0.6.4", - "rand_xoshiro", + "rand_xoshiro 0.6.0", "sized-chunks", "typenum", "version_check", @@ -3719,6 +3751,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "left-right" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc015ded5d9b3054dbbdb63332cdd6ee42352ccef19e911e25117490e2f48ee" +dependencies = [ + "crossbeam-utils", + "loom", + "slab", +] + [[package]] name = "lettre" version = "0.11.22" @@ -3855,6 +3898,19 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lopdf" version = "0.42.0" @@ -4019,6 +4075,48 @@ dependencies = [ "libc", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db0d8f1fc9e62caebd0319e11eaec5822b0186c171568f0480b46a0137f9108" +dependencies = [ + "base64 0.22.1", + "evmap", + "indexmap 2.14.0", + "metrics", + "metrics-util", + "quanta", + "thiserror 2.0.18", +] + +[[package]] +name = "metrics-util" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f8722f8562635f92f8ed992f26df0532266eb03d5202607c20c0d7e9745e13" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.4", + "rand_xoshiro 0.7.0", + "rapidhash", + "sketches-ddsketch 0.3.1", +] + [[package]] name = "mimalloc" version = "0.1.52" @@ -4481,6 +4579,8 @@ dependencies = [ "lettre", "lru", "md-5 0.11.0", + "metrics", + "metrics-exporter-prometheus", "mimalloc", "mime_guess", "mockall", @@ -4987,6 +5087,21 @@ version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-error" version = "2.0.1" @@ -5220,12 +5335,39 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "rangemap" version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -5657,6 +5799,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -5969,6 +6117,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "sketches-ddsketch" version = "0.4.0" @@ -6404,7 +6558,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", - "sketches-ddsketch", + "sketches-ddsketch 0.4.0", "smallvec", "tantivy-bitpacker", "tantivy-columnar", diff --git a/Cargo.toml b/Cargo.toml index 37d81fe7..346ba04a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,6 +134,15 @@ toml = { version = "1.1.2", optional = true } file-rotate = { version = "0.7.6", optional = true } ort = { version = "2.0.0-rc.12", default-features = false, features = ["load-dynamic", "ndarray", "tracing", "api-24"], optional = true } ndarray = { version = "0.17.2", optional = true } +# Prometheus /metrics exporter (opt-in via OXICLOUD_METRICS_LISTEN). +# `metrics` is the abstract counter API — `counter!(name, "k" => v).increment(1)` +# — and is a no-op when no recorder is installed (unset env var). The +# `metrics-exporter-prometheus` crate registers a recorder and renders the +# text-format /metrics scrape. Both re-export ordered voted versions that +# agree on their `metrics-util` transitive dep — pin as a pair when +# bumping either side. +metrics = "0.24" +metrics-exporter-prometheus = { version = "0.18", default-features = false } [features] default = [] diff --git a/docs/config/env.md b/docs/config/env.md index d8bb5d14..ffc09e5f 100644 --- a/docs/config/env.md +++ b/docs/config/env.md @@ -17,6 +17,7 @@ Most runtime variables use the `OXICLOUD_` prefix. A few build-time or allocator | `OXICLOUD_CHUNK_MAX_BYTES` | `104857600` | Maximum size of a single chunked-upload PUT in bytes (100 MB). Per-chunk cap, independent of `OXICLOUD_MAX_UPLOAD_SIZE` (whole-file cap). See [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_CHUNK_DIR` | `{STORAGE_PATH}/.uploads` | Root directory for chunked-upload sessions (REST + NextCloud). Direct (non-chunked) uploads stream straight into the blob store and need no spool directory. Placement guidance: see [Storage Fine Tuning](./storage-fine-tuning.md). | | `OXICLOUD_REUSE_PORT` | `false` | Enable `SO_REUSEPORT` so multiple processes can share the same port. **Disabled by default** — a second accidental instance will fail with "address already in use". Enable only for deliberate multi-worker setups (process supervisor, rolling restart). Not supported on Windows. | +| `OXICLOUD_METRICS_LISTEN` | (unset) | Prometheus `/metrics` listener address (e.g. `127.0.0.1:9090`, IPv6 allowed as `[::1]:9090`). **Unset = disabled**: no `/metrics` endpoint is bound and no metrics recorder is installed (zero runtime cost). When set, a separate HTTP listener on this address serves the text-format scrape. **Deliberately NOT merged into the main API** — no auth, CSRF, or DPoP layer in front. Bind to loopback or a private interface unless you intend to expose metrics publicly. Starter counters: `oxicloud_dpop_verify_failed_total{reason}`, `oxicloud_dpop_proof_missing_total`, `oxicloud_dpop_header_missing_on_bound_session_total`, `oxicloud_dpop_replay_detected_total`, `oxicloud_dpop_nonce_challenges_issued_total`. | ## Database diff --git a/example.env b/example.env index e7c86ee4..13dec374 100644 --- a/example.env +++ b/example.env @@ -42,6 +42,18 @@ OXICLOUD_SERVER_HOST=127.0.0.1 # Example: https://cloud.example.com #OXICLOUD_BASE_URL=https://cloud.example.com +# Prometheus /metrics listener — OFF by default. +# When unset (or empty), no /metrics endpoint is exposed and no +# metrics recorder is installed (zero runtime cost). +# When set, a SEPARATE HTTP listener on this address serves the +# text-format scrape at /metrics. It is NOT merged into the main +# API — no auth, CSRF, or DPoP layer in front. Bind it to loopback +# or a private interface; make it publicly reachable only if you +# intend to expose metrics publicly. +# Format: host:port (IPv6 allowed, e.g. [::1]:9090). +# Recommended: 127.0.0.1:9090 with node_exporter-style scrapers. +#OXICLOUD_METRICS_LISTEN=127.0.0.1:9090 + # ── Upload size caps ────────────────────────────────────────────────── # See docs/config/storage-fine-tuning.md for sizing guidance. diff --git a/src/common/config.rs b/src/common/config.rs index a9b62229..fc3b54bb 100644 --- a/src/common/config.rs +++ b/src/common/config.rs @@ -2533,6 +2533,16 @@ pub struct AppConfig { pub server_port: u16, /// Server host pub server_host: String, + /// Prometheus `/metrics` listener address, or `None` to disable. + /// + /// Env: `OXICLOUD_METRICS_LISTEN` (e.g. `127.0.0.1:9090`). + /// Unset / empty = no metrics recorder is installed and no + /// `/metrics` endpoint is bound (default). When set, a separate + /// axum listener on this address exposes the text-format scrape + /// — deliberately NOT merged into the main API so operators can + /// bind to loopback / a private interface without exposing + /// metrics publicly. + pub metrics_listen: Option, /// Cache configuration pub cache: CacheConfig, /// Timeout configuration @@ -2660,6 +2670,7 @@ impl Default for AppConfig { search_cache: SearchCacheConfig::default(), plugins: PluginConfig::default(), faces: FacesConfig::default(), + metrics_listen: None, } } } @@ -2691,6 +2702,22 @@ impl AppConfig { config.server_host = server_host; } + // Prometheus /metrics listener — opt-in, off by default. Empty + // string treated the same as unset (a common bare-word `=` shape + // in .env files). Parse failure is a fatal-shaped warning so + // operators don't silently ship without metrics they expected. + if let Ok(raw) = env::var("OXICLOUD_METRICS_LISTEN") + && !raw.trim().is_empty() + { + match raw.parse::() { + Ok(addr) => config.metrics_listen = Some(addr), + Err(err) => tracing::warn!( + "OXICLOUD_METRICS_LISTEN={raw:?} is not a valid socket address ({err}) \ + — metrics endpoint will NOT be exposed" + ), + } + } + // Database configuration if let Ok(connection_string) = env::var("OXICLOUD_DB_CONNECTION_STRING") { config.database.connection_string = connection_string; diff --git a/src/interfaces/metrics.rs b/src/interfaces/metrics.rs new file mode 100644 index 00000000..2ec488e3 --- /dev/null +++ b/src/interfaces/metrics.rs @@ -0,0 +1,97 @@ +//! Prometheus `/metrics` exporter — opt-in, isolated listener. +//! +//! Enabled iff `OXICLOUD_METRICS_LISTEN` is set (see +//! [`crate::common::config::AppConfig::metrics_listen`]). When unset, +//! no recorder is installed and every `metrics::counter!(…)` call +//! across the codebase compiles to a no-op — no runtime cost, no +//! endpoint bound. When set, this module: +//! +//! 1. Installs the process-global Prometheus recorder (once — panics +//! if called twice, so [`spawn`] MUST be a single-call site). +//! 2. Binds a fresh `axum` `Router` on the configured address exposing +//! only `GET /metrics`. Deliberately **not merged** into the main +//! API router — operators bind to loopback / a private interface +//! (typical: `127.0.0.1:9090` for a node_exporter-adjacent scrape) +//! without any auth, CSRF, or DPoP layer in front. Public exposure +//! is an operator choice via the bind address, not an app default. +//! 3. Spawns the listener on a detached tokio task — the metrics +//! endpoint's lifetime tracks the runtime, and a listener error +//! logs but doesn't take the main server down. +//! +//! Counter naming follows Prometheus conventions: +//! `oxicloud___total{label=…}`. Emission is +//! **duplicated** with existing audit `tracing::info!(target: "audit", …)` +//! lines — logs stay authoritative for incident forensics; counters +//! are for rate / rollup dashboards. Never remove one when adding the +//! other. +//! +//! Starter counter surface (extend as needed): +//! * `oxicloud_dpop_verify_failed_total{reason}` +//! * `oxicloud_dpop_proof_missing_total` +//! * `oxicloud_dpop_header_missing_on_bound_session_total` +//! * `oxicloud_dpop_replay_detected_total` +//! * `oxicloud_dpop_nonce_challenges_issued_total` + +use axum::{Router, extract::State, http::header, response::IntoResponse, routing::get}; +use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; +use std::net::SocketAddr; + +/// Error type returned by [`spawn`]. Uses the same `Box` +/// shape `main` already threads for setup failures — one less crate +/// dep (`anyhow`) and no coupling to a specific error framework. +pub type BoxError = Box; + +/// Install the Prometheus recorder and spawn the `/metrics` listener. +/// +/// Idempotent-unsafe: MUST be called at most once per process (the +/// recorder is a process-global singleton). Caller (main.rs) checks +/// `config.metrics_listen.is_some()` — no runtime guard here. +/// +/// Returns immediately after `bind` succeeds; the listener runs on a +/// detached tokio task. A bind failure returns the error so main can +/// decide whether to abort (recommended) or continue without metrics. +pub async fn spawn(bind: SocketAddr) -> Result<(), BoxError> { + let handle: PrometheusHandle = + PrometheusBuilder::new() + .install_recorder() + .map_err(|err| -> BoxError { + format!("failed to install Prometheus recorder: {err}").into() + })?; + + let app = Router::new() + .route("/metrics", get(scrape)) + .with_state(handle); + + let listener = tokio::net::TcpListener::bind(bind) + .await + .map_err(|err| -> BoxError { + format!("failed to bind metrics listener on {bind}: {err}").into() + })?; + let actual = listener.local_addr()?; + tracing::info!( + target: "oxicloud::metrics", + "📊 Prometheus /metrics listening on http://{actual}/metrics", + ); + + tokio::spawn(async move { + if let Err(err) = axum::serve(listener, app).await { + tracing::error!( + target: "oxicloud::metrics", + "metrics listener terminated with error: {err}", + ); + } + }); + Ok(()) +} + +/// Render the current Prometheus text-format snapshot. Content-type +/// per spec: `text/plain; version=0.0.4`; scrapers parse strictly. +async fn scrape(State(handle): State) -> impl IntoResponse { + ( + [( + header::CONTENT_TYPE, + "text/plain; version=0.0.4; charset=utf-8", + )], + handle.render(), + ) +} diff --git a/src/interfaces/middleware/dpop.rs b/src/interfaces/middleware/dpop.rs index 38bc756e..2308d8c3 100644 --- a/src/interfaces/middleware/dpop.rs +++ b/src/interfaces/middleware/dpop.rs @@ -154,6 +154,7 @@ pub async fn require_dpop_layer( user_agent = %req_user_agent, "👮🏻‍♂️ DPoP required: bound session request has no proof", ); + metrics::counter!("oxicloud_dpop_proof_missing_total").increment(1); return nonce_challenge_response(&nonce_service); } (DpopMode::Opportunistic, Some(_)) => { @@ -171,6 +172,8 @@ pub async fn require_dpop_layer( user_agent = %req_user_agent, "⚠️ DPoP: bound session sent request without a proof", ); + metrics::counter!("oxicloud_dpop_header_missing_on_bound_session_total") + .increment(1); } _ => { /* unbound session or off mode — nothing to do */ } } @@ -221,6 +224,11 @@ pub async fn require_dpop_layer( htu = %htu, "👮🏻‍♂️ DPoP nonce stale — issuing challenge", ); + metrics::counter!( + "oxicloud_dpop_verify_failed_total", + "reason" => "nonce_stale", + ) + .increment(1); return nonce_challenge_response(&nonce_service); } None => { @@ -248,6 +256,7 @@ pub async fn require_dpop_layer( jti = %verified.jti, "👮🏻‍♂️ DPoP proof replayed — same (nonce, jti) seen twice", ); + metrics::counter!("oxicloud_dpop_replay_detected_total").increment(1); return dpop_verification_failed_response( DpopVerifyError::SignatureInvalid, // shape-only; audit line carries truth &nonce_service, @@ -266,6 +275,11 @@ pub async fn require_dpop_layer( htu = %htu, "👮🏻‍♂️ DPoP proof rejected", ); + metrics::counter!( + "oxicloud_dpop_verify_failed_total", + "reason" => err.reason(), + ) + .increment(1); dpop_verification_failed_response(err, &nonce_service) } } @@ -290,9 +304,15 @@ fn stamp_current_nonce( /// WWW-Authenticate + DPoP-Nonce carrying a fresh nonce. The SPA /// fetch interceptor (Gate 4) auto-retries once with the new nonce /// so users don't experience a visible failure. +/// +/// Central counter emission (`oxicloud_dpop_nonce_challenges_issued_total`) +/// lives here rather than at each callsite — every challenge goes +/// through this helper by construction, so one increment covers all +/// three current paths (proof-missing, nonce-missing, nonce-stale). fn nonce_challenge_response( nonce_service: &crate::infrastructure::services::dpop_nonce_service::DpopNonceService, ) -> Response { + metrics::counter!("oxicloud_dpop_nonce_challenges_issued_total").increment(1); let mut resp = AppError::new( StatusCode::UNAUTHORIZED, "DPoP nonce required", diff --git a/src/interfaces/mod.rs b/src/interfaces/mod.rs index f9c9f41f..aded6acd 100644 --- a/src/interfaces/mod.rs +++ b/src/interfaces/mod.rs @@ -1,5 +1,6 @@ pub mod api; pub mod errors; +pub mod metrics; pub mod middleware; pub mod nextcloud; pub mod range_requests; diff --git a/src/main.rs b/src/main.rs index 6fd1af72..865d9e59 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1392,6 +1392,19 @@ async fn run() -> Result<(), Box> { tracing::info!("Starting OxiCloud server on http://{}", addr); + // Opt-in Prometheus `/metrics` exporter on a separate listener. + // Installs the recorder BEFORE the main listener starts serving so + // the first request's counter increments are captured (recorder + // install is racy vs first emit — order matters). + if let Some(metrics_addr) = config.metrics_listen { + if let Err(err) = oxicloud::interfaces::metrics::spawn(metrics_addr).await { + // Fail loudly: operators asked for metrics; not surfacing + // this would hide a misconfigured scrape endpoint. + tracing::error!("Prometheus /metrics setup failed: {err}"); + return Err(err); + } + } + let socket = make_socket(&addr, reuse_port)?; let listener = tokio::net::TcpListener::from_std(socket.into())?;