Files
skyg-linux-tracker/src/sources/journal.rs
T

413 lines
13 KiB
Rust

//! journald source over libsystemd.
//!
//! `systemd::journal::Journal` is neither `Send` nor `Sync`, so every
//! sd-journal call happens inside a `spawn_blocking` closure that opens its own
//! handle and drops it before returning. Nothing is held across an `.await`.
//!
//! Slow consumers: `follow` applies backpressure. The producer waits for room
//! in the bounded channel (1024 entries) and never drops entries. While
//! waiting it still notices receiver drop and shutdown.
use std::time::Duration;
use async_trait::async_trait;
use jiff::Timestamp;
use systemd::journal::{Journal, JournalRecord, OpenOptions};
use tokio::sync::{mpsc, oneshot};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use super::{clamp_count, JournalEntry, JournalQuery, JournalSource, Priority};
use crate::config::Config;
use crate::error::AppError;
const CHANNEL_CAP: usize = 1024;
const WAIT: Duration = Duration::from_millis(500);
const SEND_RETRY: Duration = Duration::from_millis(50);
pub struct SdJournal {
namespace: Option<String>,
max_lines: usize,
shutdown: CancellationToken,
}
impl SdJournal {
/// Must not fail or block.
pub fn new(config: &Config, shutdown: CancellationToken) -> Self {
let ns = config.sources.journal_namespace.trim();
Self {
namespace: (!ns.is_empty()).then(|| ns.to_string()),
max_lines: config.limits.max_lines,
shutdown,
}
}
/// Start the blocking follow loop. Returns the receiver and the join
/// handle of the blocking thread (used by tests to prove it exits).
async fn spawn_follow(
&self,
q: &JournalQuery,
) -> Result<(mpsc::Receiver<JournalEntry>, JoinHandle<()>), AppError> {
let q = q.clone();
let ns = self.namespace.clone();
let lines = clamp_count(q.lines, self.max_lines);
let token = self.shutdown.clone();
let (tx, rx) = mpsc::channel(CHANNEL_CAP);
let (ready_tx, ready_rx) = oneshot::channel();
let handle = tokio::task::spawn_blocking(move || {
let mut j = match open_filtered(ns.as_deref(), &q) {
Ok(j) => {
let _ = ready_tx.send(Ok(()));
j
}
Err(e) => {
let _ = ready_tx.send(Err(e));
return;
}
};
follow_loop(&mut j, &q, lines, &tx, &token);
});
match ready_rx.await {
Ok(Ok(())) => Ok((rx, handle)),
Ok(Err(e)) => Err(e),
Err(_) => Err(AppError::Internal("journal thread died".into())),
}
}
}
fn unavailable(e: impl std::fmt::Display) -> AppError {
AppError::Unavailable(format!("journal: {e}"))
}
fn open_filtered(ns: Option<&str>, q: &JournalQuery) -> Result<Journal, AppError> {
let opts = OpenOptions::default();
let mut j = match ns {
Some(ns) => opts.open_namespace(ns),
None => opts.open(),
}
.map_err(unavailable)?;
let plan = match_plan(q);
for (i, group) in plan.iter().enumerate() {
if i > 0 {
j.match_or().map_err(unavailable)?;
}
for (k, v) in group {
j.match_add(k, v.as_str()).map_err(unavailable)?;
}
}
Ok(j)
}
/// Disjuncts of AND-ed matches. sd-journal ANDs across fields and ORs within
/// one field, so the priority set is repeated inside each unit disjunct
/// (`_SYSTEMD_UNIT` for the unit's own lines, `UNIT` for PID1-style lines).
fn match_plan(q: &JournalQuery) -> Vec<Vec<(String, String)>> {
let prio: Vec<(String, String)> = q
.priority
.map(|p| {
(0..=p.as_u8())
.map(|k| ("PRIORITY".to_string(), k.to_string()))
.collect()
})
.unwrap_or_default();
match &q.unit {
None if prio.is_empty() => Vec::new(),
None => vec![prio],
Some(u) => ["_SYSTEMD_UNIT", "UNIT"]
.iter()
.map(|f| {
let mut g = vec![(f.to_string(), u.clone())];
g.extend(prio.iter().cloned());
g
})
.collect(),
}
}
fn to_entry(j: &mut Journal, rec: JournalRecord) -> JournalEntry {
let ts = j
.timestamp_usec()
.ok()
.and_then(|u| i64::try_from(u).ok())
.and_then(|u| Timestamp::from_microsecond(u).ok())
.unwrap_or(Timestamp::UNIX_EPOCH);
JournalEntry {
timestamp: ts,
priority: rec
.get("PRIORITY")
.and_then(|s| s.parse::<u8>().ok())
.and_then(Priority::from_u8),
unit: rec
.get("_SYSTEMD_UNIT")
.or_else(|| rec.get("UNIT"))
.cloned(),
identifier: rec.get("SYSLOG_IDENTIFIER").cloned(),
pid: rec.get("_PID").and_then(|s| s.parse().ok()),
hostname: rec.get("_HOSTNAME").cloned(),
// Values were already lossily decoded by the systemd crate.
message: rec.get("MESSAGE").cloned().unwrap_or_default(),
cursor: j.cursor().ok(),
}
}
/// Collect up to `lines` entries from a newest-first source, honoring
/// `since`, and return them oldest first. Entries with an unreadable
/// timestamp (epoch fallback) are skipped rather than ending the scan.
fn collect_backlog(
mut prev: impl FnMut() -> Result<Option<JournalEntry>, AppError>,
since: Option<Timestamp>,
lines: usize,
) -> Result<Vec<JournalEntry>, AppError> {
let mut out = Vec::new();
while out.len() < lines {
let Some(e) = prev()? else { break };
if e.timestamp == Timestamp::UNIX_EPOCH {
continue;
}
if since.is_some_and(|s| e.timestamp < s) {
break;
}
out.push(e);
}
out.reverse();
Ok(out)
}
/// Read up to `lines` most recent matching entries, oldest first.
fn read_backlog(
j: &mut Journal,
q: &JournalQuery,
lines: usize,
) -> Result<Vec<JournalEntry>, AppError> {
j.seek_tail().map_err(unavailable)?;
collect_backlog(
|| match j.previous_entry().map_err(unavailable)? {
Some(rec) => Ok(Some(to_entry(j, rec))),
None => Ok(None),
},
q.since,
lines,
)
}
/// Send with backpressure; returns false when the consumer or shutdown ended us.
fn send<T>(tx: &mpsc::Sender<T>, token: &CancellationToken, mut e: T) -> bool {
loop {
if token.is_cancelled() {
return false;
}
match tx.try_send(e) {
Ok(()) => return true,
Err(mpsc::error::TrySendError::Closed(_)) => return false,
Err(mpsc::error::TrySendError::Full(back)) => {
e = back;
std::thread::sleep(SEND_RETRY);
}
}
}
}
fn follow_loop(
j: &mut Journal,
q: &JournalQuery,
lines: usize,
tx: &mpsc::Sender<JournalEntry>,
token: &CancellationToken,
) {
let backlog = match read_backlog(j, q, lines) {
Ok(b) => b,
Err(e) => {
tracing::warn!(error = %e, "journal backlog failed");
return;
}
};
// Resume right after the newest backlog entry (or the tail if none).
let resumed = match backlog.last().and_then(|e| e.cursor.clone()) {
Some(c) => j.seek_cursor(c).and_then(|()| j.next().map(|_| ())),
None => j.seek_tail().and_then(|()| j.previous().map(|_| ())),
};
if let Err(e) = resumed {
tracing::warn!(error = %e, "journal seek failed");
return;
}
for e in backlog {
if !send(tx, token, e) {
return;
}
}
loop {
if tx.is_closed() || token.is_cancelled() {
return;
}
if let Err(e) = j.wait(Some(WAIT)) {
tracing::warn!(error = %e, "journal wait failed");
return;
}
loop {
match j.next_entry() {
Ok(Some(rec)) => {
let e = to_entry(j, rec);
if !send(tx, token, e) {
return;
}
}
Ok(None) => break,
Err(e) => {
tracing::warn!(error = %e, "journal read failed");
return;
}
}
}
}
}
#[async_trait]
impl JournalSource for SdJournal {
async fn query(&self, q: &JournalQuery) -> Result<Vec<JournalEntry>, AppError> {
let q = q.clone();
let ns = self.namespace.clone();
let lines = clamp_count(q.lines, self.max_lines);
tokio::task::spawn_blocking(move || {
let mut j = open_filtered(ns.as_deref(), &q)?;
read_backlog(&mut j, &q, lines)
})
.await
.map_err(|e| AppError::Internal(format!("journal task: {e}")))?
}
async fn follow(&self, q: &JournalQuery) -> Result<mpsc::Receiver<JournalEntry>, AppError> {
self.spawn_follow(q).await.map(|(rx, _)| rx)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn entry(sec: i64) -> JournalEntry {
JournalEntry {
timestamp: Timestamp::from_second(sec).unwrap(),
priority: None,
unit: None,
identifier: None,
pid: None,
hostname: None,
message: sec.to_string(),
cursor: None,
}
}
#[test]
fn plan_repeats_priority_in_each_unit_disjunct() {
let q = JournalQuery {
unit: Some("a.service".into()),
priority: Some(Priority::Crit),
..Default::default()
};
let plan = match_plan(&q);
assert_eq!(plan.len(), 2);
assert_eq!(plan[0][0], ("_SYSTEMD_UNIT".into(), "a.service".into()));
assert_eq!(plan[1][0], ("UNIT".into(), "a.service".into()));
for g in &plan {
let p: Vec<_> = g.iter().filter(|(k, _)| k == "PRIORITY").collect();
assert_eq!(p.len(), 3); // 0,1,2
assert_eq!(p[2].1, "2");
}
}
#[test]
fn plan_without_unit_or_priority() {
assert!(match_plan(&JournalQuery::default()).is_empty());
let q = JournalQuery {
priority: Some(Priority::Emerg),
..Default::default()
};
assert_eq!(match_plan(&q), vec![vec![("PRIORITY".into(), "0".into())]]);
}
#[test]
fn backlog_is_oldest_first_capped_and_since_filtered() {
let mut src = vec![entry(5), entry(4), entry(3), entry(2), entry(1)].into_iter();
let out = collect_backlog(|| Ok(src.next()), None, 3).unwrap();
let t: Vec<_> = out.iter().map(|e| e.message.as_str()).collect();
assert_eq!(t, ["3", "4", "5"]);
let mut src = vec![entry(5), entry(4), entry(3)].into_iter();
let since = Timestamp::from_second(4).unwrap();
let out = collect_backlog(|| Ok(src.next()), Some(since), 10).unwrap();
assert_eq!(out.len(), 2);
}
#[test]
fn backlog_skips_unreadable_timestamps() {
let mut bad = entry(0);
bad.timestamp = Timestamp::UNIX_EPOCH;
let mut src = vec![entry(3), bad, entry(1)].into_iter();
let since = Timestamp::from_second(1).unwrap();
let out = collect_backlog(|| Ok(src.next()), Some(since), 10).unwrap();
assert_eq!(out.len(), 2);
}
#[test]
fn send_stops_when_receiver_dropped() {
let (tx, rx) = mpsc::channel(1);
let token = CancellationToken::new();
assert!(send(&tx, &token, 1));
let t = std::thread::spawn({
let tx = tx.clone();
let token = token.clone();
move || send(&tx, &token, 2) // full: blocks until rx dropped
});
std::thread::sleep(Duration::from_millis(150));
drop(rx);
assert!(!t.join().unwrap());
}
#[test]
fn send_stops_on_cancel_while_full() {
let (tx, _rx) = mpsc::channel(1);
let token = CancellationToken::new();
assert!(send(&tx, &token, 1));
let t = std::thread::spawn({
let token = token.clone();
move || send(&tx, &token, 2)
});
std::thread::sleep(Duration::from_millis(150));
token.cancel();
assert!(!t.join().unwrap());
}
// Live-journal tests: `cargo test -- --ignored live_`.
#[tokio::test]
#[ignore = "needs a readable journal: cargo test -- --ignored live_"]
async fn live_follow_thread_exits_when_receiver_dropped() {
let src = SdJournal::new(&Config::default(), CancellationToken::new());
let q = JournalQuery {
lines: Some(1),
..Default::default()
};
let (rx, handle) = src.spawn_follow(&q).await.expect("journal must open");
drop(rx);
tokio::time::timeout(Duration::from_millis(1500), handle)
.await
.expect("follow thread did not exit")
.unwrap();
}
#[tokio::test]
#[ignore = "needs a readable journal: cargo test -- --ignored live_"]
async fn live_follow_thread_exits_on_shutdown() {
let token = CancellationToken::new();
let src = SdJournal::new(&Config::default(), token.clone());
let (_rx, handle) = src
.spawn_follow(&JournalQuery::default())
.await
.expect("journal must open");
token.cancel();
tokio::time::timeout(Duration::from_millis(1500), handle)
.await
.expect("follow thread did not exit")
.unwrap();
}
}