feat(telemetry): add /metrics prommetheus exporter
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
//! Prometheus `/metrics` exporter — opt-in, isolated listener.
|
||||
//!
|
||||
//! Enabled iff `OXICLOUD_METRICS_LISTEN` is set (see
|
||||
//! [`crate::common::config::AppConfig::metrics_listen`]). When unset,
|
||||
//! no recorder is installed and every `metrics::counter!(…)` call
|
||||
//! across the codebase compiles to a no-op — no runtime cost, no
|
||||
//! endpoint bound. When set, this module:
|
||||
//!
|
||||
//! 1. Installs the process-global Prometheus recorder (once — panics
|
||||
//! if called twice, so [`spawn`] MUST be a single-call site).
|
||||
//! 2. Binds a fresh `axum` `Router` on the configured address exposing
|
||||
//! only `GET /metrics`. Deliberately **not merged** into the main
|
||||
//! API router — operators bind to loopback / a private interface
|
||||
//! (typical: `127.0.0.1:9090` for a node_exporter-adjacent scrape)
|
||||
//! without any auth, CSRF, or DPoP layer in front. Public exposure
|
||||
//! is an operator choice via the bind address, not an app default.
|
||||
//! 3. Spawns the listener on a detached tokio task — the metrics
|
||||
//! endpoint's lifetime tracks the runtime, and a listener error
|
||||
//! logs but doesn't take the main server down.
|
||||
//!
|
||||
//! Counter naming follows Prometheus conventions:
|
||||
//! `oxicloud_<subsystem>_<verb>_total{label=…}`. Emission is
|
||||
//! **duplicated** with existing audit `tracing::info!(target: "audit", …)`
|
||||
//! lines — logs stay authoritative for incident forensics; counters
|
||||
//! are for rate / rollup dashboards. Never remove one when adding the
|
||||
//! other.
|
||||
//!
|
||||
//! Starter counter surface (extend as needed):
|
||||
//! * `oxicloud_dpop_verify_failed_total{reason}`
|
||||
//! * `oxicloud_dpop_proof_missing_total`
|
||||
//! * `oxicloud_dpop_header_missing_on_bound_session_total`
|
||||
//! * `oxicloud_dpop_replay_detected_total`
|
||||
//! * `oxicloud_dpop_nonce_challenges_issued_total`
|
||||
|
||||
use axum::{Router, extract::State, http::header, response::IntoResponse, routing::get};
|
||||
use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
/// Error type returned by [`spawn`]. Uses the same `Box<dyn Error>`
|
||||
/// shape `main` already threads for setup failures — one less crate
|
||||
/// dep (`anyhow`) and no coupling to a specific error framework.
|
||||
pub type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
/// Install the Prometheus recorder and spawn the `/metrics` listener.
|
||||
///
|
||||
/// Idempotent-unsafe: MUST be called at most once per process (the
|
||||
/// recorder is a process-global singleton). Caller (main.rs) checks
|
||||
/// `config.metrics_listen.is_some()` — no runtime guard here.
|
||||
///
|
||||
/// Returns immediately after `bind` succeeds; the listener runs on a
|
||||
/// detached tokio task. A bind failure returns the error so main can
|
||||
/// decide whether to abort (recommended) or continue without metrics.
|
||||
pub async fn spawn(bind: SocketAddr) -> Result<(), BoxError> {
|
||||
let handle: PrometheusHandle =
|
||||
PrometheusBuilder::new()
|
||||
.install_recorder()
|
||||
.map_err(|err| -> BoxError {
|
||||
format!("failed to install Prometheus recorder: {err}").into()
|
||||
})?;
|
||||
|
||||
let app = Router::new()
|
||||
.route("/metrics", get(scrape))
|
||||
.with_state(handle);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(bind)
|
||||
.await
|
||||
.map_err(|err| -> BoxError {
|
||||
format!("failed to bind metrics listener on {bind}: {err}").into()
|
||||
})?;
|
||||
let actual = listener.local_addr()?;
|
||||
tracing::info!(
|
||||
target: "oxicloud::metrics",
|
||||
"📊 Prometheus /metrics listening on http://{actual}/metrics",
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = axum::serve(listener, app).await {
|
||||
tracing::error!(
|
||||
target: "oxicloud::metrics",
|
||||
"metrics listener terminated with error: {err}",
|
||||
);
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Render the current Prometheus text-format snapshot. Content-type
|
||||
/// per spec: `text/plain; version=0.0.4`; scrapers parse strictly.
|
||||
async fn scrape(State(handle): State<PrometheusHandle>) -> impl IntoResponse {
|
||||
(
|
||||
[(
|
||||
header::CONTENT_TYPE,
|
||||
"text/plain; version=0.0.4; charset=utf-8",
|
||||
)],
|
||||
handle.render(),
|
||||
)
|
||||
}
|
||||
@@ -154,6 +154,7 @@ pub async fn require_dpop_layer(
|
||||
user_agent = %req_user_agent,
|
||||
"👮🏻♂️ DPoP required: bound session request has no proof",
|
||||
);
|
||||
metrics::counter!("oxicloud_dpop_proof_missing_total").increment(1);
|
||||
return nonce_challenge_response(&nonce_service);
|
||||
}
|
||||
(DpopMode::Opportunistic, Some(_)) => {
|
||||
@@ -171,6 +172,8 @@ pub async fn require_dpop_layer(
|
||||
user_agent = %req_user_agent,
|
||||
"⚠️ DPoP: bound session sent request without a proof",
|
||||
);
|
||||
metrics::counter!("oxicloud_dpop_header_missing_on_bound_session_total")
|
||||
.increment(1);
|
||||
}
|
||||
_ => { /* unbound session or off mode — nothing to do */ }
|
||||
}
|
||||
@@ -221,6 +224,11 @@ pub async fn require_dpop_layer(
|
||||
htu = %htu,
|
||||
"👮🏻♂️ DPoP nonce stale — issuing challenge",
|
||||
);
|
||||
metrics::counter!(
|
||||
"oxicloud_dpop_verify_failed_total",
|
||||
"reason" => "nonce_stale",
|
||||
)
|
||||
.increment(1);
|
||||
return nonce_challenge_response(&nonce_service);
|
||||
}
|
||||
None => {
|
||||
@@ -248,6 +256,7 @@ pub async fn require_dpop_layer(
|
||||
jti = %verified.jti,
|
||||
"👮🏻♂️ DPoP proof replayed — same (nonce, jti) seen twice",
|
||||
);
|
||||
metrics::counter!("oxicloud_dpop_replay_detected_total").increment(1);
|
||||
return dpop_verification_failed_response(
|
||||
DpopVerifyError::SignatureInvalid, // shape-only; audit line carries truth
|
||||
&nonce_service,
|
||||
@@ -266,6 +275,11 @@ pub async fn require_dpop_layer(
|
||||
htu = %htu,
|
||||
"👮🏻♂️ DPoP proof rejected",
|
||||
);
|
||||
metrics::counter!(
|
||||
"oxicloud_dpop_verify_failed_total",
|
||||
"reason" => err.reason(),
|
||||
)
|
||||
.increment(1);
|
||||
dpop_verification_failed_response(err, &nonce_service)
|
||||
}
|
||||
}
|
||||
@@ -290,9 +304,15 @@ fn stamp_current_nonce(
|
||||
/// WWW-Authenticate + DPoP-Nonce carrying a fresh nonce. The SPA
|
||||
/// fetch interceptor (Gate 4) auto-retries once with the new nonce
|
||||
/// so users don't experience a visible failure.
|
||||
///
|
||||
/// Central counter emission (`oxicloud_dpop_nonce_challenges_issued_total`)
|
||||
/// lives here rather than at each callsite — every challenge goes
|
||||
/// through this helper by construction, so one increment covers all
|
||||
/// three current paths (proof-missing, nonce-missing, nonce-stale).
|
||||
fn nonce_challenge_response(
|
||||
nonce_service: &crate::infrastructure::services::dpop_nonce_service::DpopNonceService,
|
||||
) -> Response {
|
||||
metrics::counter!("oxicloud_dpop_nonce_challenges_issued_total").increment(1);
|
||||
let mut resp = AppError::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"DPoP nonce required",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
pub mod api;
|
||||
pub mod errors;
|
||||
pub mod metrics;
|
||||
pub mod middleware;
|
||||
pub mod nextcloud;
|
||||
pub mod range_requests;
|
||||
|
||||
Reference in New Issue
Block a user