feat(telemetry): add /metrics prommetheus exporter

This commit is contained in:
Edouard Vanbelle
2026-08-09 03:05:14 +02:00
parent 80f2f67db6
commit 1b9d812175
9 changed files with 338 additions and 4 deletions
+97
View File
@@ -0,0 +1,97 @@
//! Prometheus `/metrics` exporter — opt-in, isolated listener.
//!
//! Enabled iff `OXICLOUD_METRICS_LISTEN` is set (see
//! [`crate::common::config::AppConfig::metrics_listen`]). When unset,
//! no recorder is installed and every `metrics::counter!(…)` call
//! across the codebase compiles to a no-op — no runtime cost, no
//! endpoint bound. When set, this module:
//!
//! 1. Installs the process-global Prometheus recorder (once — panics
//! if called twice, so [`spawn`] MUST be a single-call site).
//! 2. Binds a fresh `axum` `Router` on the configured address exposing
//! only `GET /metrics`. Deliberately **not merged** into the main
//! API router — operators bind to loopback / a private interface
//! (typical: `127.0.0.1:9090` for a node_exporter-adjacent scrape)
//! without any auth, CSRF, or DPoP layer in front. Public exposure
//! is an operator choice via the bind address, not an app default.
//! 3. Spawns the listener on a detached tokio task — the metrics
//! endpoint's lifetime tracks the runtime, and a listener error
//! logs but doesn't take the main server down.
//!
//! Counter naming follows Prometheus conventions:
//! `oxicloud_<subsystem>_<verb>_total{label=…}`. Emission is
//! **duplicated** with existing audit `tracing::info!(target: "audit", …)`
//! lines — logs stay authoritative for incident forensics; counters
//! are for rate / rollup dashboards. Never remove one when adding the
//! other.
//!
//! Starter counter surface (extend as needed):
//! * `oxicloud_dpop_verify_failed_total{reason}`
//! * `oxicloud_dpop_proof_missing_total`
//! * `oxicloud_dpop_header_missing_on_bound_session_total`
//! * `oxicloud_dpop_replay_detected_total`
//! * `oxicloud_dpop_nonce_challenges_issued_total`
use axum::{Router, extract::State, http::header, response::IntoResponse, routing::get};
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
use std::net::SocketAddr;
/// Error type returned by [`spawn`]. Uses the same `Box<dyn Error>`
/// shape `main` already threads for setup failures — one less crate
/// dep (`anyhow`) and no coupling to a specific error framework.
pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
/// Install the Prometheus recorder and spawn the `/metrics` listener.
///
/// Idempotent-unsafe: MUST be called at most once per process (the
/// recorder is a process-global singleton). Caller (main.rs) checks
/// `config.metrics_listen.is_some()` — no runtime guard here.
///
/// Returns immediately after `bind` succeeds; the listener runs on a
/// detached tokio task. A bind failure returns the error so main can
/// decide whether to abort (recommended) or continue without metrics.
pub async fn spawn(bind: SocketAddr) -> Result<(), BoxError> {
let handle: PrometheusHandle =
PrometheusBuilder::new()
.install_recorder()
.map_err(|err| -> BoxError {
format!("failed to install Prometheus recorder: {err}").into()
})?;
let app = Router::new()
.route("/metrics", get(scrape))
.with_state(handle);
let listener = tokio::net::TcpListener::bind(bind)
.await
.map_err(|err| -> BoxError {
format!("failed to bind metrics listener on {bind}: {err}").into()
})?;
let actual = listener.local_addr()?;
tracing::info!(
target: "oxicloud::metrics",
"📊 Prometheus /metrics listening on http://{actual}/metrics",
);
tokio::spawn(async move {
if let Err(err) = axum::serve(listener, app).await {
tracing::error!(
target: "oxicloud::metrics",
"metrics listener terminated with error: {err}",
);
}
});
Ok(())
}
/// Render the current Prometheus text-format snapshot. Content-type
/// per spec: `text/plain; version=0.0.4`; scrapers parse strictly.
async fn scrape(State(handle): State<PrometheusHandle>) -> impl IntoResponse {
(
[(
header::CONTENT_TYPE,
"text/plain; version=0.0.4; charset=utf-8",
)],
handle.render(),
)
}
+20
View File
@@ -154,6 +154,7 @@ pub async fn require_dpop_layer(
user_agent = %req_user_agent,
"👮🏻‍♂️ DPoP required: bound session request has no proof",
);
metrics::counter!("oxicloud_dpop_proof_missing_total").increment(1);
return nonce_challenge_response(&nonce_service);
}
(DpopMode::Opportunistic, Some(_)) => {
@@ -171,6 +172,8 @@ pub async fn require_dpop_layer(
user_agent = %req_user_agent,
"⚠️ DPoP: bound session sent request without a proof",
);
metrics::counter!("oxicloud_dpop_header_missing_on_bound_session_total")
.increment(1);
}
_ => { /* unbound session or off mode — nothing to do */ }
}
@@ -221,6 +224,11 @@ pub async fn require_dpop_layer(
htu = %htu,
"👮🏻‍♂️ DPoP nonce stale — issuing challenge",
);
metrics::counter!(
"oxicloud_dpop_verify_failed_total",
"reason" => "nonce_stale",
)
.increment(1);
return nonce_challenge_response(&nonce_service);
}
None => {
@@ -248,6 +256,7 @@ pub async fn require_dpop_layer(
jti = %verified.jti,
"👮🏻‍♂️ DPoP proof replayed — same (nonce, jti) seen twice",
);
metrics::counter!("oxicloud_dpop_replay_detected_total").increment(1);
return dpop_verification_failed_response(
DpopVerifyError::SignatureInvalid, // shape-only; audit line carries truth
&nonce_service,
@@ -266,6 +275,11 @@ pub async fn require_dpop_layer(
htu = %htu,
"👮🏻‍♂️ DPoP proof rejected",
);
metrics::counter!(
"oxicloud_dpop_verify_failed_total",
"reason" => err.reason(),
)
.increment(1);
dpop_verification_failed_response(err, &nonce_service)
}
}
@@ -290,9 +304,15 @@ fn stamp_current_nonce(
/// WWW-Authenticate + DPoP-Nonce carrying a fresh nonce. The SPA
/// fetch interceptor (Gate 4) auto-retries once with the new nonce
/// so users don't experience a visible failure.
///
/// Central counter emission (`oxicloud_dpop_nonce_challenges_issued_total`)
/// lives here rather than at each callsite — every challenge goes
/// through this helper by construction, so one increment covers all
/// three current paths (proof-missing, nonce-missing, nonce-stale).
fn nonce_challenge_response(
nonce_service: &crate::infrastructure::services::dpop_nonce_service::DpopNonceService,
) -> Response {
metrics::counter!("oxicloud_dpop_nonce_challenges_issued_total").increment(1);
let mut resp = AppError::new(
StatusCode::UNAUTHORIZED,
"DPoP nonce required",
+1
View File
@@ -1,5 +1,6 @@
pub mod api;
pub mod errors;
pub mod metrics;
pub mod middleware;
pub mod nextcloud;
pub mod range_requests;