//! Docker source over the unix socket via `bollard`. //! //! The client is built lazily on first use, so a missing or denied socket //! never fails startup (see ARCHITECTURE.md). Every call retries the connection until it //! succeeds, and maps backend absence to `AppError::Unavailable`. use std::io::ErrorKind; use std::path::PathBuf; use std::sync::Mutex; use async_trait::async_trait; use bollard::container::LogOutput; use bollard::errors::Error as BollardError; use bollard::models::{ContainerInspectResponse, ContainerStateStatusEnum}; use bollard::query_parameters::{ InspectContainerOptions, ListContainersOptionsBuilder, LogsOptions, LogsOptionsBuilder, StatsOptionsBuilder, }; use bollard::{Docker, API_DEFAULT_VERSION}; use futures::stream::{self, StreamExt}; use jiff::Timestamp; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; use super::{ clamp_count, ContainerDevice, ContainerResources, ContainerStats, ContainerStatus, ContainerSummary, DockerSource, LogLine, LogOptions, LogStream, }; use crate::config::Config; use crate::error::AppError; /// Request timeout for ordinary API calls, in seconds. const API_TIMEOUT_SECS: u64 = 30; /// Timeout for follow streams, which are long-lived. const FOLLOW_TIMEOUT_SECS: u64 = 24 * 60 * 60; /// Concurrent `inspect` calls during `ps`. const INSPECT_CONCURRENCY: usize = 8; /// Lines queued towards a follow consumer. const FOLLOW_CHANNEL: usize = 256; /// Log drivers the Docker API can read back. const READABLE_DRIVERS: [&str; 4] = ["json-file", "local", "journald", "k8s-file"]; pub struct BollardDocker { socket: PathBuf, max_tail: usize, shutdown: CancellationToken, client: Mutex>, } impl BollardDocker { /// Never fails or blocks. pub fn new(config: &Config, shutdown: CancellationToken) -> Self { Self { socket: config.sources.docker_socket.clone(), max_tail: config.limits.max_tail, shutdown, client: Mutex::new(None), } } fn socket_name(&self) -> String { self.socket.display().to_string() } fn connect(&self, timeout: u64) -> Result { let path = self.socket.to_string_lossy(); Docker::connect_with_unix(&path, timeout, API_DEFAULT_VERSION) .map_err(|e| map_error(&e, &self.socket_name())) } fn client(&self) -> Result { let mut guard = self.client.lock().unwrap_or_else(|p| p.into_inner()); if let Some(c) = guard.as_ref() { return Ok(c.clone()); } let c = self.connect(API_TIMEOUT_SECS)?; *guard = Some(c.clone()); Ok(c) } fn log_options(&self, opts: &LogOptions, follow: bool) -> LogsOptions { let tail = clamp_count(opts.tail, self.max_tail); let mut b = LogsOptionsBuilder::new() .follow(follow) .stdout(true) .stderr(true) .timestamps(true) .tail(&tail.to_string()); if let Some(since) = opts.since { let secs = since.as_second().clamp(0, i64::from(i32::MAX)); b = b.since(i32::try_from(secs).unwrap_or(i32::MAX)); } b.build() } } #[async_trait] impl DockerSource for BollardDocker { async fn ps(&self, all: bool) -> Result, AppError> { let docker = self.client()?; let socket = self.socket_name(); let opts = ListContainersOptionsBuilder::new().all(all).build(); let list = docker .list_containers(Some(opts)) .await .map_err(|e| map_error(&e, &socket))?; let ids: Vec = list.into_iter().filter_map(|c| c.id).collect(); // `buffered` keeps the daemon's listing order while running concurrently. let results: Vec<_> = stream::iter(ids) .map(|id| { let docker = docker.clone(); async move { docker .inspect_container(&id, None::) .await } }) .buffered(INSPECT_CONCURRENCY) .collect() .await; let now = Timestamp::now(); let mut out = Vec::with_capacity(results.len()); for r in results { match r { Ok(inspect) => out.push(summary_from_inspect(&inspect, now)), // Removed between list and inspect. Err(BollardError::DockerResponseServerError { status_code: 404, .. }) => {} Err(e) => { tracing::warn!(error = %e, "skipping container that failed to inspect"); } } } Ok(out) } async fn container_status(&self, name: &str) -> Result { let name = normalize_name(name)?; let docker = self.client()?; let inspect = inspect(&docker, &name, &self.socket_name()).await?; Ok(status_from_inspect(&inspect, Timestamp::now())) } async fn logs(&self, name: &str, opts: &LogOptions) -> Result, AppError> { let name = normalize_name(name)?; let docker = self.client()?; let socket = self.socket_name(); check_log_driver(&inspect(&docker, &name, &socket).await?, &name)?; let mut s = Box::pin(docker.logs(&name, Some(self.log_options(opts, false)))); let mut splitter = LineSplitter::default(); let mut lines = Vec::new(); while let Some(item) = s.next().await { let frame = item.map_err(|e| map_container_error(&e, &socket, &name))?; splitter.push(frame, &mut |l| lines.push(l)); } splitter.finish(&mut |l| lines.push(l)); Ok(lines) } /// The receiver closes on shutdown, consumer drop, daemon stream end or a /// stream error. End and error are indistinguishable to the consumer /// (errors are logged). A trailing unterminated line is flushed first. async fn follow_logs( &self, name: &str, opts: &LogOptions, ) -> Result, AppError> { let name = normalize_name(name)?; // Dedicated client: follow streams outlive the ordinary request timeout. let docker = self.connect(FOLLOW_TIMEOUT_SECS)?; check_log_driver(&inspect(&docker, &name, &self.socket_name()).await?, &name)?; let mut s = Box::pin(docker.logs(&name, Some(self.log_options(opts, true)))); let (tx, rx) = mpsc::channel(FOLLOW_CHANNEL); let shutdown = self.shutdown.clone(); tokio::spawn(async move { let mut splitter = LineSplitter::default(); 'outer: loop { let next = tokio::select! { biased; () = shutdown.cancelled() => break, () = tx.closed() => break, next = s.next() => next, }; let mut batch = Vec::new(); let mut finished = false; match next { Some(Ok(frame)) => splitter.push(frame, &mut |l| batch.push(l)), Some(Err(e)) => { tracing::warn!(error = %e, container = %name, "docker log follow ended"); finished = true; } None => finished = true, } if finished { splitter.finish(&mut |l| batch.push(l)); } for line in batch { tokio::select! { biased; () = shutdown.cancelled() => break 'outer, sent = tx.send(line) => { if sent.is_err() { break 'outer; } } } } if finished { break; } } // Dropping `s` closes the HTTP connection and ends the daemon-side stream. }); Ok(rx) } async fn stats(&self, name: &str) -> Result { let name = normalize_name(name)?; let docker = self.client()?; let socket = self.socket_name(); // One shot: the daemon samples once, answers and closes the stream. let opts = StatsOptionsBuilder::new() .stream(false) .one_shot(true) .build(); let mut s = Box::pin(docker.stats(&name, Some(opts))); let sample = match s.next().await { Some(Ok(sample)) => sample, Some(Err(e)) => return Err(map_container_error(&e, &socket, &name)), None => { return Err(AppError::Internal(format!( "docker returned no stats for '{name}'" ))) } }; Ok(stats_from_response(&sample, Timestamp::now())) } } /// Strip the leading `/` and reject names that could alter the request path. pub(crate) fn normalize_name(name: &str) -> Result { let n = name.trim().trim_start_matches('/'); let valid = n.len() <= 255 && n.chars().next().is_some_and(|c| c.is_ascii_alphanumeric()) && n.chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-')); if valid { Ok(n.to_owned()) } else { Err(AppError::BadRequest(format!( "invalid container name '{name}'" ))) } } async fn inspect( docker: &Docker, name: &str, socket: &str, ) -> Result { docker .inspect_container(name, None::) .await .map_err(|e| map_container_error(&e, socket, name)) } /// Reject log reads for drivers the API cannot serve, naming the driver. fn check_log_driver(i: &ContainerInspectResponse, name: &str) -> Result<(), AppError> { let driver = i .host_config .as_ref() .and_then(|h| h.log_config.as_ref()) .and_then(|l| l.typ.as_deref()) .unwrap_or(""); if driver.is_empty() || READABLE_DRIVERS.contains(&driver) { Ok(()) } else { Err(AppError::BadRequest(format!( "logs unavailable for '{name}': log driver '{driver}' cannot be read through the Docker API" ))) } } /// Map a bollard error that is not specific to one container. fn map_error(e: &BollardError, socket: &str) -> AppError { match e { BollardError::DockerResponseServerError { status_code, message, } => match *status_code { 401 | 403 => AppError::Unavailable(format!("docker denied access: {message}")), 404 => AppError::NotFound(message.clone()), 400 => AppError::BadRequest(message.clone()), 501 => AppError::BadRequest(message.clone()), 500 if message.to_ascii_lowercase().contains("logging driver") => { AppError::BadRequest(message.clone()) } _ => AppError::Internal(format!("docker API error {status_code}: {message}")), }, BollardError::IOError { err } => match err.kind() { ErrorKind::NotFound => { AppError::Unavailable(format!("docker socket {socket} not found")) } ErrorKind::PermissionDenied => { AppError::Unavailable(format!("permission denied on docker socket {socket}")) } ErrorKind::ConnectionRefused => { AppError::Unavailable(format!("connection refused on docker socket {socket}")) } _ => AppError::Unavailable(format!("docker socket {socket}: {err}")), }, BollardError::SocketNotFoundError(_) => { AppError::Unavailable(format!("docker socket {socket} not found")) } BollardError::RequestTimeoutError => { AppError::Unavailable(format!("docker socket {socket}: request timed out")) } // Transport-level failures mean the daemon is unreachable. BollardError::HyperResponseError { .. } | BollardError::HyperLegacyError { .. } | BollardError::HttpClientError { .. } => { AppError::Unavailable(format!("docker socket {socket} unreachable: {e}")) } other => AppError::Internal(format!("docker: {other}")), } } fn map_container_error(e: &BollardError, socket: &str, name: &str) -> AppError { match e { BollardError::DockerResponseServerError { status_code: 404, .. } => AppError::NotFound(format!("container '{name}' not found")), other => map_error(other, socket), } } fn parse_ts(s: &str) -> Option { // Never-started containers report year 0001. s.parse::().ok().filter(|t| t.as_second() > 0) } fn summary_from_inspect(i: &ContainerInspectResponse, now: Timestamp) -> ContainerSummary { let state = i.state.as_ref(); let state_name = state .and_then(|s| s.status) .map_or("unknown", state_str) .to_owned(); let health = state .and_then(|s| s.health.as_ref()) .and_then(|h| h.status) .map(|h| h.to_string()) .filter(|h| !h.is_empty() && h != "none"); let started_at = state .and_then(|s| s.started_at.as_deref()) .and_then(parse_ts); let uptime_seconds = match (state_name == "running", started_at) { (true, Some(t)) => { Some(u64::try_from((now.as_second() - t.as_second()).max(0)).unwrap_or(0)) } _ => None, }; let image = i .config .as_ref() .and_then(|c| c.image.clone()) .filter(|s| !s.is_empty()) .or_else(|| i.image.clone()) .unwrap_or_default(); ContainerSummary { id: i.id.clone().unwrap_or_default(), name: i .name .as_deref() .unwrap_or_default() .trim_start_matches('/') .to_owned(), image, state: state_name, health, created: i.created.as_ref().and_then(|c| c.to_string().parse().ok()), started_at, uptime_seconds, restart_count: i.restart_count.and_then(|n| u64::try_from(n).ok()), } } fn status_from_inspect(i: &ContainerInspectResponse, now: Timestamp) -> ContainerStatus { let state = i.state.as_ref(); let health_output = state .and_then(|s| s.health.as_ref()) .and_then(|h| h.log.as_ref()) .and_then(|l| l.last()) .and_then(|r| r.output.clone()); ContainerStatus { summary: summary_from_inspect(i, now), finished_at: state .and_then(|s| s.finished_at.as_deref()) .and_then(parse_ts), exit_code: state.and_then(|s| s.exit_code), oom_killed: state.and_then(|s| s.oom_killed), health_output, image_id: i.image.clone().filter(|s| !s.is_empty()), resources: i.host_config.as_ref().map(resources_from_host_config), } } /// Copy only limits and sandbox flags. `Env`, `Labels`, `Cmd`, `Args` and mounts /// stay on the daemon, so no credential can travel through this API. fn resources_from_host_config(h: &bollard::models::HostConfig) -> ContainerResources { ContainerResources { cpu_cores: cpu_cores_from(h.nano_cpus, h.cpu_quota, h.cpu_period), cpu_shares: h.cpu_shares, memory_bytes: h.memory.filter(|v| *v > 0), memory_swap_bytes: h.memory_swap.filter(|v| *v > 0), pids_limit: h.pids_limit.filter(|v| *v > 0), privileged: h.privileged, read_only_rootfs: h.readonly_rootfs, network_mode: h.network_mode.clone().filter(|s| !s.is_empty()), restart_policy: h .restart_policy .as_ref() .and_then(|p| p.name) .map(|n| n.to_string()) .filter(|s| !s.is_empty()), devices: h .device_requests .as_deref() .unwrap_or_default() .iter() .map(device_from_request) .collect(), } } /// `nano_cpus` wins; a quota/period pair is the older way to say the same thing. /// Non-positive values mean "unlimited" in both spellings. fn cpu_cores_from(nano_cpus: Option, quota: Option, period: Option) -> Option { if let Some(n) = nano_cpus.filter(|n| *n > 0) { return Some(n as f64 / 1_000_000_000.0); } let (q, p) = (quota.filter(|q| *q > 0)?, period.filter(|p| *p > 0)?); Some(q as f64 / p as f64) } fn device_from_request(r: &bollard::models::DeviceRequest) -> ContainerDevice { ContainerDevice { driver: r.driver.clone().filter(|s| !s.is_empty()), count: r.count, ids: r.device_ids.clone().unwrap_or_default(), capabilities: r .capabilities .clone() .unwrap_or_default() .into_iter() .flatten() .collect(), } } fn stats_from_response( s: &bollard::models::ContainerStatsResponse, now: Timestamp, ) -> ContainerStats { let mem = s.memory_stats.as_ref(); let page_cache = mem .and_then(|m| m.stats.as_ref()) .and_then(|st| st.get("inactive_file").or_else(|| st.get("cache"))) .copied(); let used = mem .and_then(|m| m.usage) .map(|u| u.saturating_sub(page_cache.unwrap_or(0))); let limit = mem.and_then(|m| m.limit).filter(|l| *l > 0); ContainerStats { sampled_at: now, cpu_percent: cpu_percent_from(s), memory_used_bytes: used, memory_limit_bytes: limit, memory_percent: match (used, limit) { (Some(u), Some(l)) => Some(u as f64 / l as f64 * 100.0), _ => None, }, pids: s.pids_stats.as_ref().and_then(|p| p.current), block_read_bytes: blkio_bytes(s, "read"), block_write_bytes: blkio_bytes(s, "write"), network_rx_bytes: network_bytes(s, true), network_tx_bytes: network_bytes(s, false), } } /// Percent of one core; 400 means four cores saturated. Needs the previous /// sample the daemon sends alongside, so a zero delta answers `None`. fn cpu_percent_from(s: &bollard::models::ContainerStatsResponse) -> Option { let cpu = s.cpu_stats.as_ref()?; let pre = s.precpu_stats.as_ref()?; let total = cpu.cpu_usage.as_ref()?.total_usage?; let pre_total = pre.cpu_usage.as_ref()?.total_usage?; let system = cpu.system_cpu_usage?; let pre_system = pre.system_cpu_usage?; let cpu_delta = total.checked_sub(pre_total)? as f64; let system_delta = system.checked_sub(pre_system)? as f64; if system_delta <= 0.0 { return None; } let cores = cpu .online_cpus .filter(|n| *n > 0) .map(f64::from) .or_else(|| { cpu.cpu_usage .as_ref() .and_then(|u| u.percpu_usage.as_ref()) .map(|v| v.len() as f64) }) .unwrap_or(1.0); Some(cpu_delta / system_delta * cores * 100.0) } fn blkio_bytes(s: &bollard::models::ContainerStatsResponse, op: &str) -> Option { let entries = s .blkio_stats .as_ref() .and_then(|b| b.io_service_bytes_recursive.as_ref())?; let total: u64 = entries .iter() .filter(|e| e.op.as_deref().is_some_and(|o| o.eq_ignore_ascii_case(op))) .filter_map(|e| e.value) .sum(); Some(total) } fn network_bytes(s: &bollard::models::ContainerStatsResponse, rx: bool) -> Option { let nets = s.networks.as_ref()?; Some( nets.values() .filter_map(|n| if rx { n.rx_bytes } else { n.tx_bytes }) .sum(), ) } fn state_str(s: ContainerStateStatusEnum) -> &'static str { use ContainerStateStatusEnum as S; match s { S::EMPTY => "unknown", S::CREATED => "created", S::RUNNING => "running", S::PAUSED => "paused", S::RESTARTING => "restarting", S::REMOVING => "removing", S::EXITED => "exited", S::DEAD => "dead", S::STOPPING => "stopping", } } /// Reassembles log frames into lines, one buffer per stream. #[derive(Default)] struct LineSplitter { out: Vec, err: Vec, } impl LineSplitter { fn push(&mut self, frame: LogOutput, emit: &mut dyn FnMut(LogLine)) { let (stream, buf, bytes) = match frame { LogOutput::StdErr { message } => (LogStream::Stderr, &mut self.err, message), LogOutput::StdOut { message } | LogOutput::Console { message } => { (LogStream::Stdout, &mut self.out, message) } LogOutput::StdIn { .. } => return, }; buf.extend_from_slice(&bytes); while let Some(pos) = buf.iter().position(|b| *b == b'\n') { let line: Vec = buf.drain(..=pos).collect(); emit(parse_line(stream, &line[..line.len() - 1])); } } /// Flush any unterminated trailing lines. fn finish(&mut self, emit: &mut dyn FnMut(LogLine)) { for (stream, buf) in [ (LogStream::Stdout, &mut self.out), (LogStream::Stderr, &mut self.err), ] { if !buf.is_empty() { emit(parse_line(stream, buf)); buf.clear(); } } } } /// Parse `" "` produced by `timestamps=true`. fn parse_line(stream: LogStream, raw: &[u8]) -> LogLine { let text = String::from_utf8_lossy(raw); let text = text.strip_suffix('\r').unwrap_or(&text); if let Some((head, rest)) = text.split_once(' ') { if let Ok(ts) = head.parse::() { return LogLine { stream, ts: Some(ts), message: rest.to_owned(), }; } } LogLine { stream, ts: None, message: text.to_owned(), } } #[cfg(test)] #[path = "docker_tests.rs"] mod tests;