feat(job-registry): add the job engine
This commit is contained in:
Edouard Vanbelle
@@ -2,4 +2,5 @@ pub mod adapters;
|
||||
pub mod auth_factory;
|
||||
pub mod db;
|
||||
pub mod repositories;
|
||||
pub mod scheduler;
|
||||
pub mod services;
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
//! The scheduler supervisor loop.
|
||||
//!
|
||||
//! One `tokio::spawn` at startup runs [`SchedulerEngine::run`]. The
|
||||
//! loop iterates:
|
||||
//!
|
||||
//! 1. `pick_next()` — find the job with the earliest `next_run_at`.
|
||||
//! 2. Sleep until that instant.
|
||||
//! 3. Dispatch: try-acquire the job's in-flight permit; if held, warn
|
||||
//! and reschedule; otherwise spawn the handler, apply the timeout,
|
||||
//! catch panics, record the outcome.
|
||||
//!
|
||||
//! Sequential dispatch is intentional. Two jobs due at the same
|
||||
//! instant run one-after-the-other — the second's `pick_next` fires
|
||||
//! immediately after the first's dispatch returns, with a zero-length
|
||||
//! sleep. See `docs/plan/job-registry.md` Part 1 §Runtime model.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use chrono::Utc;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use super::registry::{JobEntry, JobRegistry};
|
||||
use super::types::{ErrCause, JobOutcome};
|
||||
|
||||
/// Public handle to the running supervisor.
|
||||
///
|
||||
/// Dropping the handle does NOT cancel the loop (the spawned task
|
||||
/// runs until the runtime dies). Explicit shutdown is deferred to
|
||||
/// whenever graceful-shutdown lands globally — matches the shape
|
||||
/// every other daemon in the codebase has today. See
|
||||
/// `docs/plan/job-registry.md` Part 1 §Shutdown coordination.
|
||||
pub struct SchedulerEngine {
|
||||
_handle: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl SchedulerEngine {
|
||||
/// Spawn the supervisor loop and return a handle. Callers hold
|
||||
/// the returned `SchedulerEngine` on `AppState` so the task lives
|
||||
/// as long as the runtime.
|
||||
pub fn start(registry: Arc<JobRegistry>) -> Self {
|
||||
let handle = tokio::spawn(async move {
|
||||
run(registry).await;
|
||||
});
|
||||
Self { _handle: handle }
|
||||
}
|
||||
}
|
||||
|
||||
/// If the registry is empty (no jobs registered yet), sleep this long
|
||||
/// before rechecking. Registration happens once at boot in the current
|
||||
/// design, so this only matters as a defensive fallback — in practice
|
||||
/// the loop enters this branch at most once, right before the first
|
||||
/// `register()` call completes.
|
||||
const IDLE_POLL: Duration = Duration::from_secs(60);
|
||||
|
||||
async fn run(registry: Arc<JobRegistry>) {
|
||||
tracing::info!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "scheduler.started",
|
||||
"periodic scheduler supervisor started"
|
||||
);
|
||||
|
||||
loop {
|
||||
let Some((name, next_at)) = registry.pick_next().await else {
|
||||
tokio::time::sleep(IDLE_POLL).await;
|
||||
continue;
|
||||
};
|
||||
|
||||
// Convert to `Duration`. If `next_at` is in the past (missed
|
||||
// tick, e.g. very short interval and the previous dispatch
|
||||
// took longer than the interval), sleep zero and dispatch
|
||||
// immediately.
|
||||
let now = Utc::now();
|
||||
let sleep_dur = (next_at - now)
|
||||
.to_std()
|
||||
.unwrap_or_else(|_| Duration::from_millis(0));
|
||||
if !sleep_dur.is_zero() {
|
||||
tokio::time::sleep(sleep_dur).await;
|
||||
}
|
||||
|
||||
// The job's `next_run_at` might have changed since `pick_next`
|
||||
// returned if a concurrent trigger fired — that's fine; the
|
||||
// dispatch below re-reads via the `JobEntry` snapshot.
|
||||
let Some(entry) = registry.get(&name).await else {
|
||||
// Job was unregistered between pick_next and dispatch —
|
||||
// unreachable in the current design (no unregister), but
|
||||
// guard defensively.
|
||||
continue;
|
||||
};
|
||||
|
||||
dispatch(&name, entry).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Dispatch a single tick for `name`. Handles:
|
||||
/// - exclusivity: try-acquire the in-flight permit; skip + warn if held,
|
||||
/// - spawning under panic containment (via `tokio::spawn` + `JoinHandle`),
|
||||
/// - timeout enforcement (if `ScheduledJob.timeout` is set),
|
||||
/// - recording `last_outcome` + advancing `next_run_at` on completion.
|
||||
///
|
||||
/// Non-panicking; every failure path resolves to a `JobOutcome::Err`
|
||||
/// with a `cause` log field.
|
||||
async fn dispatch(name: &str, entry: Arc<JobEntry>) {
|
||||
// Try to acquire the single-permit gate. `try_acquire` is
|
||||
// non-blocking — if held, we know the previous run is still
|
||||
// executing and skip this tick.
|
||||
let permit = match entry.in_flight.try_acquire() {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
// Someone else holds the permit → previous run still in
|
||||
// flight. Emit the operator-signal warning and reschedule.
|
||||
let running_for_ms = {
|
||||
let state = entry.state.lock().expect("JobState mutex poisoned");
|
||||
state
|
||||
.current_run_start
|
||||
.map(|t| t.elapsed().as_millis())
|
||||
.unwrap_or(0)
|
||||
};
|
||||
tracing::warn!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "job.tick_skipped",
|
||||
job = %name,
|
||||
interval_ms = entry.interval.as_millis(),
|
||||
running_for_ms = running_for_ms,
|
||||
"{} still running past its interval — tick skipped",
|
||||
name,
|
||||
);
|
||||
advance_next_run(&entry);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// We hold the permit. Record run-start, spawn, await, translate.
|
||||
{
|
||||
let mut state = entry.state.lock().expect("JobState mutex poisoned");
|
||||
state.current_run_start = Some(Instant::now());
|
||||
}
|
||||
let started_wall = Utc::now();
|
||||
let start_instant = Instant::now();
|
||||
|
||||
// Spawn so panics land as `JoinError::is_panic()` instead of
|
||||
// unwinding into the supervisor loop.
|
||||
let handler = entry.handler.clone();
|
||||
let join = tokio::spawn(async move { handler.run().await });
|
||||
|
||||
let (outcome, cause) = match entry.timeout {
|
||||
Some(dur) => match tokio::time::timeout(dur, join).await {
|
||||
Ok(res) => translate_join(res),
|
||||
Err(_elapsed) => {
|
||||
// Timeout fired. The JoinHandle is dropped, which
|
||||
// aborts the spawned task cooperatively — but abort
|
||||
// is best-effort in Rust; a handler that ignores
|
||||
// yield points may run to completion in the background.
|
||||
// We still record timeout and release the permit.
|
||||
(
|
||||
JobOutcome::Err(format!("wall-clock timeout of {:?} exceeded", dur)),
|
||||
Some(ErrCause::Timeout),
|
||||
)
|
||||
}
|
||||
},
|
||||
None => translate_join(join.await),
|
||||
};
|
||||
|
||||
let elapsed_ms = start_instant.elapsed().as_millis();
|
||||
|
||||
// Record outcome and advance the schedule. Permit drops naturally
|
||||
// when `permit` goes out of scope at the end of the function.
|
||||
{
|
||||
let mut state = entry.state.lock().expect("JobState mutex poisoned");
|
||||
state.current_run_start = None;
|
||||
state.last_outcome = Some((started_wall, outcome.clone()));
|
||||
// Same rule as the skip branch — schedule advances by one
|
||||
// interval, no backlog queueing. If this run took longer than
|
||||
// the interval, the next `pick_next` immediately sees a past
|
||||
// due time and dispatches with zero sleep, but the exclusivity
|
||||
// check keeps the in-flight guarantee intact.
|
||||
state.next_run_at = Utc::now()
|
||||
+ chrono::Duration::from_std(entry.interval)
|
||||
.unwrap_or_else(|_| chrono::Duration::seconds(0));
|
||||
}
|
||||
|
||||
// Log line. `outcome=ok` runs are informational; `outcome=err` include
|
||||
// the diagnostic `cause` field.
|
||||
log_outcome(name, &outcome, cause, elapsed_ms);
|
||||
|
||||
drop(permit);
|
||||
}
|
||||
|
||||
/// Advance `next_run_at` by one interval without touching outcome or
|
||||
/// run-start (skip-path helper). Keeps the schedule steady rather
|
||||
/// than queueing missed ticks.
|
||||
fn advance_next_run(entry: &JobEntry) {
|
||||
let mut state = entry.state.lock().expect("JobState mutex poisoned");
|
||||
state.next_run_at = Utc::now()
|
||||
+ chrono::Duration::from_std(entry.interval)
|
||||
.unwrap_or_else(|_| chrono::Duration::seconds(0));
|
||||
}
|
||||
|
||||
/// Convert the `Result<JobOutcome, JoinError>` returned by the spawned
|
||||
/// handler into `(JobOutcome, Option<ErrCause>)`. `cause` is `None`
|
||||
/// on Ok, `Some(_)` on Err.
|
||||
fn translate_join(
|
||||
res: Result<JobOutcome, tokio::task::JoinError>,
|
||||
) -> (JobOutcome, Option<ErrCause>) {
|
||||
match res {
|
||||
Ok(outcome) => {
|
||||
let cause = if outcome.is_ok() {
|
||||
None
|
||||
} else {
|
||||
Some(ErrCause::Handler)
|
||||
};
|
||||
(outcome, cause)
|
||||
}
|
||||
Err(join_err) if join_err.is_panic() => {
|
||||
let payload = join_err.into_panic();
|
||||
let msg = if let Some(s) = payload.downcast_ref::<&'static str>() {
|
||||
(*s).to_string()
|
||||
} else if let Some(s) = payload.downcast_ref::<String>() {
|
||||
s.clone()
|
||||
} else {
|
||||
"unknown panic payload".to_string()
|
||||
};
|
||||
(
|
||||
JobOutcome::Err(format!("handler panicked: {msg}")),
|
||||
Some(ErrCause::Panicked),
|
||||
)
|
||||
}
|
||||
Err(join_err) => (
|
||||
JobOutcome::Err(format!("task cancelled: {join_err}")),
|
||||
Some(ErrCause::Handler),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Emit the uniform `oxicloud::scheduler` log line for a completed run.
|
||||
/// Distinct Ok/Err branches so the tracing macros pick up the fields at
|
||||
/// compile time — `tracing` doesn't expand conditional field lists.
|
||||
fn log_outcome(name: &str, outcome: &JobOutcome, cause: Option<ErrCause>, elapsed_ms: u128) {
|
||||
match outcome {
|
||||
JobOutcome::Ok { count, extra } => {
|
||||
tracing::info!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "job.run",
|
||||
job = %name,
|
||||
outcome = "ok",
|
||||
count = *count,
|
||||
elapsed_ms = elapsed_ms,
|
||||
extra = %extra,
|
||||
"job {} ran",
|
||||
name,
|
||||
);
|
||||
}
|
||||
JobOutcome::Err(msg) => {
|
||||
tracing::warn!(
|
||||
target: "oxicloud::scheduler",
|
||||
event = "job.run",
|
||||
job = %name,
|
||||
outcome = "err",
|
||||
cause = %cause.unwrap_or(ErrCause::Handler),
|
||||
elapsed_ms = elapsed_ms,
|
||||
error = %msg,
|
||||
"job {} failed",
|
||||
name,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::infrastructure::scheduler::handler::JobHandler;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
struct CountingHandler {
|
||||
name: String,
|
||||
calls: Arc<AtomicU64>,
|
||||
sleep: Duration,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for CountingHandler {
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
async fn run(&self) -> JobOutcome {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if !self.sleep.is_zero() {
|
||||
tokio::time::sleep(self.sleep).await;
|
||||
}
|
||||
JobOutcome::ok(1)
|
||||
}
|
||||
}
|
||||
|
||||
struct PanickingHandler;
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for PanickingHandler {
|
||||
fn name(&self) -> &str {
|
||||
"panicker"
|
||||
}
|
||||
async fn run(&self) -> JobOutcome {
|
||||
panic!("intentional test panic");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn panic_containment_via_translate_join() {
|
||||
// Directly exercise translate_join with a spawned panic — the
|
||||
// supervisor loop's dispatch path uses this same helper.
|
||||
let handler = Arc::new(PanickingHandler);
|
||||
let join = tokio::spawn(async move { handler.run().await });
|
||||
let (outcome, cause) = translate_join(join.await);
|
||||
assert!(!outcome.is_ok());
|
||||
assert_eq!(cause, Some(ErrCause::Panicked));
|
||||
if let JobOutcome::Err(msg) = outcome {
|
||||
assert!(msg.contains("panicked"), "expected panic marker in: {msg}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn overrun_skips_second_tick() {
|
||||
// Handler that sleeps 200 ms; two dispatches fired back-to-back
|
||||
// should see the second skip with a `tick_skipped` warning.
|
||||
let calls = Arc::new(AtomicU64::new(0));
|
||||
let handler = Arc::new(CountingHandler {
|
||||
name: "overrun".to_string(),
|
||||
calls: calls.clone(),
|
||||
sleep: Duration::from_millis(200),
|
||||
});
|
||||
|
||||
let registry = Arc::new(JobRegistry::new());
|
||||
registry
|
||||
.register(handler, Duration::from_millis(100), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let entry = registry.get("overrun").await.unwrap();
|
||||
|
||||
// Kick off dispatch 1 in the background — it holds the permit
|
||||
// for ~200 ms.
|
||||
let entry_bg = entry.clone();
|
||||
let bg = tokio::spawn(async move { dispatch("overrun", entry_bg).await });
|
||||
|
||||
// Give dispatch 1 time to grab the permit.
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
|
||||
// Dispatch 2 should observe the permit taken and skip.
|
||||
dispatch("overrun", entry.clone()).await;
|
||||
|
||||
// Only dispatch 1's handler should have actually run so far.
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
bg.await.unwrap();
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timeout_records_err_and_releases_permit() {
|
||||
let calls = Arc::new(AtomicU64::new(0));
|
||||
let handler = Arc::new(CountingHandler {
|
||||
name: "slow".to_string(),
|
||||
calls: calls.clone(),
|
||||
sleep: Duration::from_millis(500),
|
||||
});
|
||||
|
||||
let registry = Arc::new(JobRegistry::new());
|
||||
registry
|
||||
.register(
|
||||
handler,
|
||||
Duration::from_millis(100),
|
||||
Some(Duration::from_millis(50)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let entry = registry.get("slow").await.unwrap();
|
||||
|
||||
dispatch("slow", entry.clone()).await;
|
||||
|
||||
// The timeout fired; last_outcome must be Err.
|
||||
let state = entry.state.lock().unwrap();
|
||||
let (_, outcome) = state.last_outcome.as_ref().expect("outcome recorded");
|
||||
assert!(!outcome.is_ok(), "expected timeout-Err, got {outcome:?}");
|
||||
|
||||
// Permit released — another dispatch could acquire it.
|
||||
assert_eq!(entry.in_flight.available_permits(), 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
//! The implementor-facing contract for a Part 1 (periodic) job.
|
||||
//!
|
||||
//! Everything a native service needs to write to plug into the periodic
|
||||
//! scheduler is on this page. See `docs/plan/job-registry.md` Part 1
|
||||
//! for the design rationale and migration criterion (the "operator
|
||||
//! trigger" question — if an operator would never `POST /trigger-job`
|
||||
//! for this loop, it doesn't belong here; keep it as a core worker).
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use super::types::JobOutcome;
|
||||
|
||||
/// Implemented by every service that wants to run on a fixed interval
|
||||
/// through the periodic scheduler.
|
||||
///
|
||||
/// # Design shape
|
||||
///
|
||||
/// A single method, `run()`. One tick = one call. The supervisor:
|
||||
/// - fires it at the registered interval,
|
||||
/// - catches panics (bad handlers crash their own run, not the scheduler),
|
||||
/// - enforces the configured wall-clock timeout (if any),
|
||||
/// - enforces exclusivity — a second tick that fires while a previous
|
||||
/// run is still executing is **skipped, not queued**, with a
|
||||
/// `job.tick_skipped` warning emitted (the operator signal that the
|
||||
/// job is chronically slower than its cadence).
|
||||
///
|
||||
/// Implementors write the body of `run()`. Everything else — logging,
|
||||
/// timing, panic containment, exclusivity — is the supervisor's job.
|
||||
///
|
||||
/// # `name()` guidance
|
||||
///
|
||||
/// Return a stable, unique snake_case identifier. Log lines
|
||||
/// (`job = %name`), admin listing, admin trigger URLs
|
||||
/// (`POST /api/admin/internal/trigger-job/{name}`) and env vars
|
||||
/// (`OXICLOUD_JOB_<NAME>_INTERVAL_HOURS`) all key on this. Renaming
|
||||
/// after release is a breaking change to operator scripts and log
|
||||
/// dashboards.
|
||||
///
|
||||
/// # `run()` guidance
|
||||
///
|
||||
/// Return [`JobOutcome::Ok`] with a `count` scalar the operator finds
|
||||
/// meaningful (rows swept, blobs GC'd, bytes reclaimed) plus optional
|
||||
/// `extra` JSON. Return [`JobOutcome::Err`] on failure — the
|
||||
/// supervisor logs it under `outcome=err, cause=handler` and moves
|
||||
/// on; the next tick fires normally.
|
||||
///
|
||||
/// **Do not** catch panics inside `run()` — the supervisor does it,
|
||||
/// and hiding one loses the `cause=panicked` diagnostic signal.
|
||||
///
|
||||
/// **Do not** call `tokio::time::sleep` for long durations inside
|
||||
/// `run()` if you have a `timeout` configured — the timeout fires
|
||||
/// mid-sleep and kills the run with `cause=timeout`. Use short polling
|
||||
/// intervals or restructure the work.
|
||||
///
|
||||
/// # Reference implementation
|
||||
///
|
||||
/// See `TrashCleanupService::run` (once migrated) as the canonical
|
||||
/// example: reads its own configuration, runs bounded work, returns
|
||||
/// a count. Everything else is boilerplate the scheduler owns.
|
||||
#[async_trait]
|
||||
pub trait JobHandler: Send + Sync {
|
||||
/// Stable, unique snake_case identifier. Must be unique across
|
||||
/// the process; the registry rejects duplicate registration.
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// One execution. Called at the registered interval and (optionally)
|
||||
/// on admin trigger. See trait-level docs for guidance on when to
|
||||
/// return Ok vs Err.
|
||||
async fn run(&self) -> JobOutcome;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
//! Periodic job scheduler (Part 1 of the job-registry plan).
|
||||
//!
|
||||
//! In-memory registration + one-supervisor-task dispatch. Fires
|
||||
//! registered [`JobHandler`] implementations at their configured
|
||||
//! intervals with panic containment, timeout enforcement, and
|
||||
//! same-name exclusivity.
|
||||
//!
|
||||
//! # For future implementors
|
||||
//!
|
||||
//! - **You write a `JobHandler`.** Implement [`JobHandler::name`] +
|
||||
//! [`JobHandler::run`] on your service. Nothing else. See
|
||||
//! [`handler`] for the guidance doc-comment.
|
||||
//! - **DI wires the registration.** In `common/di.rs` (or wherever the
|
||||
//! composition root lives), build an `Arc<JobRegistry>` once, register
|
||||
//! every service that opts in, then call [`SchedulerEngine::start`].
|
||||
//! - **Should this loop actually be a scheduler job?** See the migration
|
||||
//! criterion in `docs/plan/job-registry.md` — the primary question
|
||||
//! is "would an operator plausibly trigger this manually?". Continuous
|
||||
//! drains and event-reactive workers stay as their own loops.
|
||||
//!
|
||||
//! Part 2 (recoverable-run engine, DB-backed cursor + resume) is
|
||||
//! designed but not yet implemented. When it lands it will slot in
|
||||
//! as a sibling module without changing anything here.
|
||||
|
||||
mod engine;
|
||||
mod handler;
|
||||
mod registry;
|
||||
mod types;
|
||||
|
||||
pub use engine::SchedulerEngine;
|
||||
pub use handler::JobHandler;
|
||||
pub use registry::{JobEntry, JobRegistry, RegisterError};
|
||||
pub use types::{ErrCause, JobOutcome};
|
||||
@@ -0,0 +1,239 @@
|
||||
//! In-memory registry of periodic jobs.
|
||||
//!
|
||||
//! The [`JobRegistry`] owns a map `name → JobEntry`. Native services
|
||||
//! `register()` themselves during DI; the [`SchedulerEngine`](super::engine::SchedulerEngine)
|
||||
//! iterates this map on every tick to pick the next-due job.
|
||||
//!
|
||||
//! Per-job state (in-flight semaphore, last outcome, next-run time)
|
||||
//! lives inside each [`JobEntry`] behind a short-lived `std::sync::Mutex`.
|
||||
//! The outer map uses a `tokio::sync::RwLock` so `pick_next` and
|
||||
//! `snapshot` don't block one another and so future dynamic
|
||||
//! registration (plugin manifests, admin UI) can acquire a write
|
||||
//! lock without racing readers.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
|
||||
use super::handler::JobHandler;
|
||||
use super::types::JobOutcome;
|
||||
|
||||
/// A registered job plus its runtime state. Held as `Arc<JobEntry>`
|
||||
/// inside the registry so the engine can hold a snapshot across an
|
||||
/// `await` without pinning the registry's outer lock.
|
||||
pub struct JobEntry {
|
||||
pub(super) handler: Arc<dyn JobHandler>,
|
||||
pub(super) interval: Duration,
|
||||
pub(super) timeout: Option<Duration>,
|
||||
/// Single-permit gate enforcing the "one in-flight run per
|
||||
/// `job_name`" invariant. A tick that finds the permit taken
|
||||
/// emits `job.tick_skipped` and does not spawn.
|
||||
pub(super) in_flight: Semaphore,
|
||||
/// Mutable state — protected by `std::sync::Mutex` because guards
|
||||
/// are only held for a few statements at a time, never across an
|
||||
/// `await`. `tokio::sync::Mutex` would add overhead for no benefit.
|
||||
pub(super) state: Mutex<JobState>,
|
||||
}
|
||||
|
||||
pub(super) struct JobState {
|
||||
/// Set when a run starts, cleared when it ends. Used to include
|
||||
/// `running_for_ms` in the `job.tick_skipped` warning.
|
||||
pub current_run_start: Option<Instant>,
|
||||
/// Wall-clock time + outcome of the most recent completed run.
|
||||
/// `None` until the first run finishes.
|
||||
pub last_outcome: Option<(DateTime<Utc>, JobOutcome)>,
|
||||
/// Wall-clock time of the next scheduled dispatch. Advanced by
|
||||
/// one interval after every tick — both successful dispatch and
|
||||
/// skipped (in-flight) tick.
|
||||
pub next_run_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// In-memory job registry. `Arc<JobRegistry>` lives on `AppState`;
|
||||
/// native services `register()` during DI wiring.
|
||||
pub struct JobRegistry {
|
||||
entries: RwLock<HashMap<String, Arc<JobEntry>>>,
|
||||
}
|
||||
|
||||
impl JobRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
entries: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a job. Returns an error if a job with the same name
|
||||
/// is already registered — names are the primary identifier
|
||||
/// everywhere (logs, admin URLs, env vars) and collisions would
|
||||
/// hide bugs.
|
||||
///
|
||||
/// `first_run_at` = `Utc::now() + interval` by convention —
|
||||
/// registration doesn't fire the job immediately. Callers that
|
||||
/// want an at-startup run should invoke the service's own
|
||||
/// initialiser once before registering.
|
||||
pub async fn register(
|
||||
&self,
|
||||
handler: Arc<dyn JobHandler>,
|
||||
interval: Duration,
|
||||
timeout: Option<Duration>,
|
||||
) -> Result<(), RegisterError> {
|
||||
let name = handler.name().to_string();
|
||||
let mut guard = self.entries.write().await;
|
||||
if guard.contains_key(&name) {
|
||||
return Err(RegisterError::DuplicateName(name));
|
||||
}
|
||||
let now = Utc::now();
|
||||
let next_run_at = now
|
||||
+ chrono::Duration::from_std(interval).unwrap_or_else(|_| chrono::Duration::seconds(0));
|
||||
let entry = Arc::new(JobEntry {
|
||||
handler,
|
||||
interval,
|
||||
timeout,
|
||||
in_flight: Semaphore::new(1),
|
||||
state: Mutex::new(JobState {
|
||||
current_run_start: None,
|
||||
last_outcome: None,
|
||||
next_run_at,
|
||||
}),
|
||||
});
|
||||
guard.insert(name, entry);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return the name and next-due timestamp of the job that fires
|
||||
/// soonest, or `None` if no jobs are registered. Read-lock only —
|
||||
/// safe to call frequently from the supervisor loop.
|
||||
pub async fn pick_next(&self) -> Option<(String, DateTime<Utc>)> {
|
||||
let guard = self.entries.read().await;
|
||||
let mut earliest: Option<(String, DateTime<Utc>)> = None;
|
||||
for (name, entry) in guard.iter() {
|
||||
let next_at = entry
|
||||
.state
|
||||
.lock()
|
||||
.expect("JobState mutex poisoned")
|
||||
.next_run_at;
|
||||
match &earliest {
|
||||
None => earliest = Some((name.clone(), next_at)),
|
||||
Some((_, current)) if next_at < *current => {
|
||||
earliest = Some((name.clone(), next_at))
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
earliest
|
||||
}
|
||||
|
||||
/// Snapshot handle to a single job. Returns `Arc<JobEntry>` so
|
||||
/// callers can hold across `await` points without pinning the
|
||||
/// outer read lock.
|
||||
pub async fn get(&self, name: &str) -> Option<Arc<JobEntry>> {
|
||||
let guard = self.entries.read().await;
|
||||
guard.get(name).cloned()
|
||||
}
|
||||
|
||||
/// Snapshot every registered job (used by the admin listing
|
||||
/// endpoint). Returns owned `(name, Arc<JobEntry>)` pairs to
|
||||
/// avoid pinning the outer lock through the HTTP response
|
||||
/// serialisation.
|
||||
pub async fn snapshot_all(&self) -> Vec<(String, Arc<JobEntry>)> {
|
||||
let guard = self.entries.read().await;
|
||||
guard.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
|
||||
}
|
||||
|
||||
/// Count of registered jobs — used for the startup log line.
|
||||
pub async fn len(&self) -> usize {
|
||||
self.entries.read().await.len()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub async fn is_empty(&self) -> bool {
|
||||
self.entries.read().await.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for JobRegistry {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RegisterError {
|
||||
#[error("job name already registered: {0}")]
|
||||
DuplicateName(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use async_trait::async_trait;
|
||||
|
||||
struct DummyHandler {
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for DummyHandler {
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
async fn run(&self) -> JobOutcome {
|
||||
JobOutcome::ok(0)
|
||||
}
|
||||
}
|
||||
|
||||
fn handler(name: &str) -> Arc<dyn JobHandler> {
|
||||
Arc::new(DummyHandler {
|
||||
name: name.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn register_and_pick_next() {
|
||||
let reg = JobRegistry::new();
|
||||
reg.register(handler("job_a"), Duration::from_secs(60), None)
|
||||
.await
|
||||
.unwrap();
|
||||
reg.register(handler("job_b"), Duration::from_secs(10), None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (next_name, _) = reg.pick_next().await.expect("expected a due job");
|
||||
// job_b has the shorter interval → earlier next_run_at.
|
||||
assert_eq!(next_name, "job_b");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn duplicate_registration_rejected() {
|
||||
let reg = JobRegistry::new();
|
||||
reg.register(handler("job_x"), Duration::from_secs(60), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let err = reg
|
||||
.register(handler("job_x"), Duration::from_secs(60), None)
|
||||
.await
|
||||
.expect_err("duplicate name must be rejected");
|
||||
assert!(matches!(err, RegisterError::DuplicateName(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_registry_picks_nothing() {
|
||||
let reg = JobRegistry::new();
|
||||
assert!(reg.pick_next().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snapshot_all_returns_every_entry() {
|
||||
let reg = JobRegistry::new();
|
||||
reg.register(handler("a"), Duration::from_secs(1), None)
|
||||
.await
|
||||
.unwrap();
|
||||
reg.register(handler("b"), Duration::from_secs(1), None)
|
||||
.await
|
||||
.unwrap();
|
||||
let all = reg.snapshot_all().await;
|
||||
assert_eq!(all.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
//! Common types shared across the scheduler module.
|
||||
//!
|
||||
//! Nothing here talks to the DB or the async runtime — pure data
|
||||
//! definitions so downstream modules (handler, registry, engine) can
|
||||
//! import without dragging in transitive dependencies. See
|
||||
//! `docs/plan/job-registry.md` Part 1 for the design rationale.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Uniform outcome the supervisor logs and stores for every job dispatch.
|
||||
///
|
||||
/// Two variants, deliberately. Distinguishing *why* a job failed
|
||||
/// (handler returned Err, `tokio::time::timeout` tripped,
|
||||
/// `catch_unwind` caught a panic) is a **diagnostic** concern — it
|
||||
/// belongs in a `cause` tracing field the supervisor sets, not in a
|
||||
/// control-flow branch every consumer of `match outcome` has to
|
||||
/// think about. See `docs/plan/job-registry.md` Part 1 §JobOutcome.
|
||||
///
|
||||
/// `Ok::count` is the row/record count the job reports as its primary
|
||||
/// scalar (rows scanned, blobs migrated, thumbnails checked). `extra`
|
||||
/// is a free-form JSON blob for job-specific fields the caller wants
|
||||
/// surfaced to `oxicloud::scheduler` log lines.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "outcome", rename_all = "snake_case")]
|
||||
pub enum JobOutcome {
|
||||
Ok {
|
||||
count: u64,
|
||||
#[serde(default, skip_serializing_if = "serde_json::Value::is_null")]
|
||||
extra: serde_json::Value,
|
||||
},
|
||||
Err(String),
|
||||
}
|
||||
|
||||
impl JobOutcome {
|
||||
/// Ok with no extras — the common case for jobs that only report a count.
|
||||
pub fn ok(count: u64) -> Self {
|
||||
JobOutcome::Ok {
|
||||
count,
|
||||
extra: serde_json::Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
/// Ok with a JSON `extra` payload. Use `serde_json::json!({...})`
|
||||
/// at call sites for readability.
|
||||
pub fn ok_with(count: u64, extra: serde_json::Value) -> Self {
|
||||
JobOutcome::Ok { count, extra }
|
||||
}
|
||||
|
||||
/// Terse discriminant for logs / metrics: `"ok"` | `"err"`.
|
||||
pub fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
JobOutcome::Ok { .. } => "ok",
|
||||
JobOutcome::Err(_) => "err",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_ok(&self) -> bool {
|
||||
matches!(self, JobOutcome::Ok { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// Diagnostic reason the supervisor attaches to the `cause` tracing
|
||||
/// field when a job's outcome is [`JobOutcome::Err`]. Never persisted
|
||||
/// as a first-class column — it's a log field only.
|
||||
///
|
||||
/// Handlers never construct this; the supervisor derives it from
|
||||
/// which failure path fired:
|
||||
/// - [`ErrCause::Handler`] — the handler returned `Err(_)` itself.
|
||||
/// - [`ErrCause::Timeout`] — `tokio::time::timeout` tripped on the
|
||||
/// registered `ScheduledJob.timeout` wall-clock cap.
|
||||
/// - [`ErrCause::Panicked`] — `JoinHandle` returned a panic error.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ErrCause {
|
||||
Handler,
|
||||
Timeout,
|
||||
Panicked,
|
||||
}
|
||||
|
||||
impl ErrCause {
|
||||
/// Stable label for the `cause` tracing field. Log aggregators key
|
||||
/// on these — renaming here IS a breaking change to any dashboard
|
||||
/// filtering on `cause = "handler"`.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
ErrCause::Handler => "handler",
|
||||
ErrCause::Timeout => "timeout",
|
||||
ErrCause::Panicked => "panicked",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ErrCause {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn joboutcome_kind_label() {
|
||||
assert_eq!(JobOutcome::ok(0).kind(), "ok");
|
||||
assert_eq!(JobOutcome::Err("boom".into()).kind(), "err");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn errcause_labels_stable() {
|
||||
assert_eq!(ErrCause::Handler.as_str(), "handler");
|
||||
assert_eq!(ErrCause::Timeout.as_str(), "timeout");
|
||||
assert_eq!(ErrCause::Panicked.as_str(), "panicked");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user