//! journald source over libsystemd. //! //! `systemd::journal::Journal` is neither `Send` nor `Sync`, so every //! sd-journal call happens inside a `spawn_blocking` closure that opens its own //! handle and drops it before returning. Nothing is held across an `.await`. //! //! Slow consumers: `follow` applies backpressure. The producer waits for room //! in the bounded channel (1024 entries) and never drops entries. While //! waiting it still notices receiver drop and shutdown. use std::time::Duration; use async_trait::async_trait; use jiff::Timestamp; use systemd::journal::{Journal, JournalRecord, OpenOptions}; use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use super::{clamp_count, JournalEntry, JournalQuery, JournalSource, Priority}; use crate::config::Config; use crate::error::AppError; const CHANNEL_CAP: usize = 1024; const WAIT: Duration = Duration::from_millis(500); const SEND_RETRY: Duration = Duration::from_millis(50); pub struct SdJournal { namespace: Option, max_lines: usize, shutdown: CancellationToken, } impl SdJournal { /// Must not fail or block. pub fn new(config: &Config, shutdown: CancellationToken) -> Self { let ns = config.sources.journal_namespace.trim(); Self { namespace: (!ns.is_empty()).then(|| ns.to_string()), max_lines: config.limits.max_lines, shutdown, } } /// Start the blocking follow loop. Returns the receiver and the join /// handle of the blocking thread (used by tests to prove it exits). async fn spawn_follow( &self, q: &JournalQuery, ) -> Result<(mpsc::Receiver, JoinHandle<()>), AppError> { let q = q.clone(); let ns = self.namespace.clone(); let lines = clamp_count(q.lines, self.max_lines); let token = self.shutdown.clone(); let (tx, rx) = mpsc::channel(CHANNEL_CAP); let (ready_tx, ready_rx) = oneshot::channel(); let handle = tokio::task::spawn_blocking(move || { let mut j = match open_filtered(ns.as_deref(), &q) { Ok(j) => { let _ = ready_tx.send(Ok(())); j } Err(e) => { let _ = ready_tx.send(Err(e)); return; } }; follow_loop(&mut j, &q, lines, &tx, &token); }); match ready_rx.await { Ok(Ok(())) => Ok((rx, handle)), Ok(Err(e)) => Err(e), Err(_) => Err(AppError::Internal("journal thread died".into())), } } } fn unavailable(e: impl std::fmt::Display) -> AppError { AppError::Unavailable(format!("journal: {e}")) } fn open_filtered(ns: Option<&str>, q: &JournalQuery) -> Result { let opts = OpenOptions::default(); let mut j = match ns { Some(ns) => opts.open_namespace(ns), None => opts.open(), } .map_err(unavailable)?; let plan = match_plan(q); for (i, group) in plan.iter().enumerate() { if i > 0 { j.match_or().map_err(unavailable)?; } for (k, v) in group { j.match_add(k, v.as_str()).map_err(unavailable)?; } } Ok(j) } /// Disjuncts of AND-ed matches. sd-journal ANDs across fields and ORs within /// one field, so the priority set is repeated inside each unit disjunct /// (`_SYSTEMD_UNIT` for the unit's own lines, `UNIT` for PID1-style lines). fn match_plan(q: &JournalQuery) -> Vec> { let prio: Vec<(String, String)> = q .priority .map(|p| { (0..=p.as_u8()) .map(|k| ("PRIORITY".to_string(), k.to_string())) .collect() }) .unwrap_or_default(); match &q.unit { None if prio.is_empty() => Vec::new(), None => vec![prio], Some(u) => ["_SYSTEMD_UNIT", "UNIT"] .iter() .map(|f| { let mut g = vec![(f.to_string(), u.clone())]; g.extend(prio.iter().cloned()); g }) .collect(), } } fn to_entry(j: &mut Journal, rec: JournalRecord) -> JournalEntry { let ts = j .timestamp_usec() .ok() .and_then(|u| i64::try_from(u).ok()) .and_then(|u| Timestamp::from_microsecond(u).ok()) .unwrap_or(Timestamp::UNIX_EPOCH); JournalEntry { timestamp: ts, priority: rec .get("PRIORITY") .and_then(|s| s.parse::().ok()) .and_then(Priority::from_u8), unit: rec .get("_SYSTEMD_UNIT") .or_else(|| rec.get("UNIT")) .cloned(), identifier: rec.get("SYSLOG_IDENTIFIER").cloned(), pid: rec.get("_PID").and_then(|s| s.parse().ok()), hostname: rec.get("_HOSTNAME").cloned(), // Values were already lossily decoded by the systemd crate. message: rec.get("MESSAGE").cloned().unwrap_or_default(), cursor: j.cursor().ok(), } } /// Collect up to `lines` entries from a newest-first source, honoring /// `since`, and return them oldest first. Entries with an unreadable /// timestamp (epoch fallback) are skipped rather than ending the scan. fn collect_backlog( mut prev: impl FnMut() -> Result, AppError>, since: Option, lines: usize, ) -> Result, AppError> { let mut out = Vec::new(); while out.len() < lines { let Some(e) = prev()? else { break }; if e.timestamp == Timestamp::UNIX_EPOCH { continue; } if since.is_some_and(|s| e.timestamp < s) { break; } out.push(e); } out.reverse(); Ok(out) } /// Read up to `lines` most recent matching entries, oldest first. fn read_backlog( j: &mut Journal, q: &JournalQuery, lines: usize, ) -> Result, AppError> { j.seek_tail().map_err(unavailable)?; collect_backlog( || match j.previous_entry().map_err(unavailable)? { Some(rec) => Ok(Some(to_entry(j, rec))), None => Ok(None), }, q.since, lines, ) } /// Send with backpressure; returns false when the consumer or shutdown ended us. fn send(tx: &mpsc::Sender, token: &CancellationToken, mut e: T) -> bool { loop { if token.is_cancelled() { return false; } match tx.try_send(e) { Ok(()) => return true, Err(mpsc::error::TrySendError::Closed(_)) => return false, Err(mpsc::error::TrySendError::Full(back)) => { e = back; std::thread::sleep(SEND_RETRY); } } } } fn follow_loop( j: &mut Journal, q: &JournalQuery, lines: usize, tx: &mpsc::Sender, token: &CancellationToken, ) { let backlog = match read_backlog(j, q, lines) { Ok(b) => b, Err(e) => { tracing::warn!(error = %e, "journal backlog failed"); return; } }; // Resume right after the newest backlog entry (or the tail if none). let resumed = match backlog.last().and_then(|e| e.cursor.clone()) { Some(c) => j.seek_cursor(c).and_then(|()| j.next().map(|_| ())), None => j.seek_tail().and_then(|()| j.previous().map(|_| ())), }; if let Err(e) = resumed { tracing::warn!(error = %e, "journal seek failed"); return; } for e in backlog { if !send(tx, token, e) { return; } } loop { if tx.is_closed() || token.is_cancelled() { return; } if let Err(e) = j.wait(Some(WAIT)) { tracing::warn!(error = %e, "journal wait failed"); return; } loop { match j.next_entry() { Ok(Some(rec)) => { let e = to_entry(j, rec); if !send(tx, token, e) { return; } } Ok(None) => break, Err(e) => { tracing::warn!(error = %e, "journal read failed"); return; } } } } } #[async_trait] impl JournalSource for SdJournal { async fn query(&self, q: &JournalQuery) -> Result, AppError> { let q = q.clone(); let ns = self.namespace.clone(); let lines = clamp_count(q.lines, self.max_lines); tokio::task::spawn_blocking(move || { let mut j = open_filtered(ns.as_deref(), &q)?; read_backlog(&mut j, &q, lines) }) .await .map_err(|e| AppError::Internal(format!("journal task: {e}")))? } async fn follow(&self, q: &JournalQuery) -> Result, AppError> { self.spawn_follow(q).await.map(|(rx, _)| rx) } } #[cfg(test)] mod tests { use super::*; fn entry(sec: i64) -> JournalEntry { JournalEntry { timestamp: Timestamp::from_second(sec).unwrap(), priority: None, unit: None, identifier: None, pid: None, hostname: None, message: sec.to_string(), cursor: None, } } #[test] fn plan_repeats_priority_in_each_unit_disjunct() { let q = JournalQuery { unit: Some("a.service".into()), priority: Some(Priority::Crit), ..Default::default() }; let plan = match_plan(&q); assert_eq!(plan.len(), 2); assert_eq!(plan[0][0], ("_SYSTEMD_UNIT".into(), "a.service".into())); assert_eq!(plan[1][0], ("UNIT".into(), "a.service".into())); for g in &plan { let p: Vec<_> = g.iter().filter(|(k, _)| k == "PRIORITY").collect(); assert_eq!(p.len(), 3); // 0,1,2 assert_eq!(p[2].1, "2"); } } #[test] fn plan_without_unit_or_priority() { assert!(match_plan(&JournalQuery::default()).is_empty()); let q = JournalQuery { priority: Some(Priority::Emerg), ..Default::default() }; assert_eq!(match_plan(&q), vec![vec![("PRIORITY".into(), "0".into())]]); } #[test] fn backlog_is_oldest_first_capped_and_since_filtered() { let mut src = vec![entry(5), entry(4), entry(3), entry(2), entry(1)].into_iter(); let out = collect_backlog(|| Ok(src.next()), None, 3).unwrap(); let t: Vec<_> = out.iter().map(|e| e.message.as_str()).collect(); assert_eq!(t, ["3", "4", "5"]); let mut src = vec![entry(5), entry(4), entry(3)].into_iter(); let since = Timestamp::from_second(4).unwrap(); let out = collect_backlog(|| Ok(src.next()), Some(since), 10).unwrap(); assert_eq!(out.len(), 2); } #[test] fn backlog_skips_unreadable_timestamps() { let mut bad = entry(0); bad.timestamp = Timestamp::UNIX_EPOCH; let mut src = vec![entry(3), bad, entry(1)].into_iter(); let since = Timestamp::from_second(1).unwrap(); let out = collect_backlog(|| Ok(src.next()), Some(since), 10).unwrap(); assert_eq!(out.len(), 2); } #[test] fn send_stops_when_receiver_dropped() { let (tx, rx) = mpsc::channel(1); let token = CancellationToken::new(); assert!(send(&tx, &token, 1)); let t = std::thread::spawn({ let tx = tx.clone(); let token = token.clone(); move || send(&tx, &token, 2) // full: blocks until rx dropped }); std::thread::sleep(Duration::from_millis(150)); drop(rx); assert!(!t.join().unwrap()); } #[test] fn send_stops_on_cancel_while_full() { let (tx, _rx) = mpsc::channel(1); let token = CancellationToken::new(); assert!(send(&tx, &token, 1)); let t = std::thread::spawn({ let token = token.clone(); move || send(&tx, &token, 2) }); std::thread::sleep(Duration::from_millis(150)); token.cancel(); assert!(!t.join().unwrap()); } // Live-journal tests: `cargo test -- --ignored live_`. #[tokio::test] #[ignore = "needs a readable journal: cargo test -- --ignored live_"] async fn live_follow_thread_exits_when_receiver_dropped() { let src = SdJournal::new(&Config::default(), CancellationToken::new()); let q = JournalQuery { lines: Some(1), ..Default::default() }; let (rx, handle) = src.spawn_follow(&q).await.expect("journal must open"); drop(rx); tokio::time::timeout(Duration::from_millis(1500), handle) .await .expect("follow thread did not exit") .unwrap(); } #[tokio::test] #[ignore = "needs a readable journal: cargo test -- --ignored live_"] async fn live_follow_thread_exits_on_shutdown() { let token = CancellationToken::new(); let src = SdJournal::new(&Config::default(), token.clone()); let (_rx, handle) = src .spawn_follow(&JournalQuery::default()) .await .expect("journal must open"); token.cancel(); tokio::time::timeout(Duration::from_millis(1500), handle) .await .expect("follow thread did not exit") .unwrap(); } }