Files
skyg-linux-tracker/src/sources/docker.rs
T
HermenixandClaude Sonnet 5.5 0ff3520563 container details: limits, sandbox flags and a live stats sample
Container details now carry the image id and the HostConfig resource limits
(CPU cores and shares, memory and swap bytes, pids, device requests) plus sandbox flags
(privileged, read-only rootfs, network mode, restart policy), with null meaning unlimited. A new
GET /api/v1/docker/containers/{name}/stats returns one live sample: CPU percent
of one core, memory working set, processes, block IO and network bytes. Peers
proxy the new route.

Env, labels, command lines and mounts are deliberately excluded from the payload.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-05 03:02:33 +00:00

619 lines
22 KiB
Rust

//! Docker source over the unix socket via `bollard`.
//!
//! The client is built lazily on first use, so a missing or denied socket
//! never fails startup (see ARCHITECTURE.md). Every call retries the connection until it
//! succeeds, and maps backend absence to `AppError::Unavailable`.
use std::io::ErrorKind;
use std::path::PathBuf;
use std::sync::Mutex;
use async_trait::async_trait;
use bollard::container::LogOutput;
use bollard::errors::Error as BollardError;
use bollard::models::{ContainerInspectResponse, ContainerStateStatusEnum};
use bollard::query_parameters::{
InspectContainerOptions, ListContainersOptionsBuilder, LogsOptions, LogsOptionsBuilder,
StatsOptionsBuilder,
};
use bollard::{Docker, API_DEFAULT_VERSION};
use futures::stream::{self, StreamExt};
use jiff::Timestamp;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use super::{
clamp_count, ContainerDevice, ContainerResources, ContainerStats, ContainerStatus,
ContainerSummary, DockerSource, LogLine, LogOptions, LogStream,
};
use crate::config::Config;
use crate::error::AppError;
/// Request timeout for ordinary API calls, in seconds.
const API_TIMEOUT_SECS: u64 = 30;
/// Timeout for follow streams, which are long-lived.
const FOLLOW_TIMEOUT_SECS: u64 = 24 * 60 * 60;
/// Concurrent `inspect` calls during `ps`.
const INSPECT_CONCURRENCY: usize = 8;
/// Lines queued towards a follow consumer.
const FOLLOW_CHANNEL: usize = 256;
/// Log drivers the Docker API can read back.
const READABLE_DRIVERS: [&str; 4] = ["json-file", "local", "journald", "k8s-file"];
pub struct BollardDocker {
socket: PathBuf,
max_tail: usize,
shutdown: CancellationToken,
client: Mutex<Option<Docker>>,
}
impl BollardDocker {
/// Never fails or blocks.
pub fn new(config: &Config, shutdown: CancellationToken) -> Self {
Self {
socket: config.sources.docker_socket.clone(),
max_tail: config.limits.max_tail,
shutdown,
client: Mutex::new(None),
}
}
fn socket_name(&self) -> String {
self.socket.display().to_string()
}
fn connect(&self, timeout: u64) -> Result<Docker, AppError> {
let path = self.socket.to_string_lossy();
Docker::connect_with_unix(&path, timeout, API_DEFAULT_VERSION)
.map_err(|e| map_error(&e, &self.socket_name()))
}
fn client(&self) -> Result<Docker, AppError> {
let mut guard = self.client.lock().unwrap_or_else(|p| p.into_inner());
if let Some(c) = guard.as_ref() {
return Ok(c.clone());
}
let c = self.connect(API_TIMEOUT_SECS)?;
*guard = Some(c.clone());
Ok(c)
}
fn log_options(&self, opts: &LogOptions, follow: bool) -> LogsOptions {
let tail = clamp_count(opts.tail, self.max_tail);
let mut b = LogsOptionsBuilder::new()
.follow(follow)
.stdout(true)
.stderr(true)
.timestamps(true)
.tail(&tail.to_string());
if let Some(since) = opts.since {
let secs = since.as_second().clamp(0, i64::from(i32::MAX));
b = b.since(i32::try_from(secs).unwrap_or(i32::MAX));
}
b.build()
}
}
#[async_trait]
impl DockerSource for BollardDocker {
async fn ps(&self, all: bool) -> Result<Vec<ContainerSummary>, AppError> {
let docker = self.client()?;
let socket = self.socket_name();
let opts = ListContainersOptionsBuilder::new().all(all).build();
let list = docker
.list_containers(Some(opts))
.await
.map_err(|e| map_error(&e, &socket))?;
let ids: Vec<String> = list.into_iter().filter_map(|c| c.id).collect();
// `buffered` keeps the daemon's listing order while running concurrently.
let results: Vec<_> = stream::iter(ids)
.map(|id| {
let docker = docker.clone();
async move {
docker
.inspect_container(&id, None::<InspectContainerOptions>)
.await
}
})
.buffered(INSPECT_CONCURRENCY)
.collect()
.await;
let now = Timestamp::now();
let mut out = Vec::with_capacity(results.len());
for r in results {
match r {
Ok(inspect) => out.push(summary_from_inspect(&inspect, now)),
// Removed between list and inspect.
Err(BollardError::DockerResponseServerError {
status_code: 404, ..
}) => {}
Err(e) => {
tracing::warn!(error = %e, "skipping container that failed to inspect");
}
}
}
Ok(out)
}
async fn container_status(&self, name: &str) -> Result<ContainerStatus, AppError> {
let name = normalize_name(name)?;
let docker = self.client()?;
let inspect = inspect(&docker, &name, &self.socket_name()).await?;
Ok(status_from_inspect(&inspect, Timestamp::now()))
}
async fn logs(&self, name: &str, opts: &LogOptions) -> Result<Vec<LogLine>, AppError> {
let name = normalize_name(name)?;
let docker = self.client()?;
let socket = self.socket_name();
check_log_driver(&inspect(&docker, &name, &socket).await?, &name)?;
let mut s = Box::pin(docker.logs(&name, Some(self.log_options(opts, false))));
let mut splitter = LineSplitter::default();
let mut lines = Vec::new();
while let Some(item) = s.next().await {
let frame = item.map_err(|e| map_container_error(&e, &socket, &name))?;
splitter.push(frame, &mut |l| lines.push(l));
}
splitter.finish(&mut |l| lines.push(l));
Ok(lines)
}
/// The receiver closes on shutdown, consumer drop, daemon stream end or a
/// stream error. End and error are indistinguishable to the consumer
/// (errors are logged). A trailing unterminated line is flushed first.
async fn follow_logs(
&self,
name: &str,
opts: &LogOptions,
) -> Result<mpsc::Receiver<LogLine>, AppError> {
let name = normalize_name(name)?;
// Dedicated client: follow streams outlive the ordinary request timeout.
let docker = self.connect(FOLLOW_TIMEOUT_SECS)?;
check_log_driver(&inspect(&docker, &name, &self.socket_name()).await?, &name)?;
let mut s = Box::pin(docker.logs(&name, Some(self.log_options(opts, true))));
let (tx, rx) = mpsc::channel(FOLLOW_CHANNEL);
let shutdown = self.shutdown.clone();
tokio::spawn(async move {
let mut splitter = LineSplitter::default();
'outer: loop {
let next = tokio::select! {
biased;
() = shutdown.cancelled() => break,
() = tx.closed() => break,
next = s.next() => next,
};
let mut batch = Vec::new();
let mut finished = false;
match next {
Some(Ok(frame)) => splitter.push(frame, &mut |l| batch.push(l)),
Some(Err(e)) => {
tracing::warn!(error = %e, container = %name, "docker log follow ended");
finished = true;
}
None => finished = true,
}
if finished {
splitter.finish(&mut |l| batch.push(l));
}
for line in batch {
tokio::select! {
biased;
() = shutdown.cancelled() => break 'outer,
sent = tx.send(line) => {
if sent.is_err() {
break 'outer;
}
}
}
}
if finished {
break;
}
}
// Dropping `s` closes the HTTP connection and ends the daemon-side stream.
});
Ok(rx)
}
async fn stats(&self, name: &str) -> Result<ContainerStats, AppError> {
let name = normalize_name(name)?;
let docker = self.client()?;
let socket = self.socket_name();
// One shot: the daemon samples once, answers and closes the stream.
let opts = StatsOptionsBuilder::new()
.stream(false)
.one_shot(true)
.build();
let mut s = Box::pin(docker.stats(&name, Some(opts)));
let sample = match s.next().await {
Some(Ok(sample)) => sample,
Some(Err(e)) => return Err(map_container_error(&e, &socket, &name)),
None => {
return Err(AppError::Internal(format!(
"docker returned no stats for '{name}'"
)))
}
};
Ok(stats_from_response(&sample, Timestamp::now()))
}
}
/// Strip the leading `/` and reject names that could alter the request path.
pub(crate) fn normalize_name(name: &str) -> Result<String, AppError> {
let n = name.trim().trim_start_matches('/');
let valid = n.len() <= 255
&& n.chars().next().is_some_and(|c| c.is_ascii_alphanumeric())
&& n.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-'));
if valid {
Ok(n.to_owned())
} else {
Err(AppError::BadRequest(format!(
"invalid container name '{name}'"
)))
}
}
async fn inspect(
docker: &Docker,
name: &str,
socket: &str,
) -> Result<ContainerInspectResponse, AppError> {
docker
.inspect_container(name, None::<InspectContainerOptions>)
.await
.map_err(|e| map_container_error(&e, socket, name))
}
/// Reject log reads for drivers the API cannot serve, naming the driver.
fn check_log_driver(i: &ContainerInspectResponse, name: &str) -> Result<(), AppError> {
let driver = i
.host_config
.as_ref()
.and_then(|h| h.log_config.as_ref())
.and_then(|l| l.typ.as_deref())
.unwrap_or("");
if driver.is_empty() || READABLE_DRIVERS.contains(&driver) {
Ok(())
} else {
Err(AppError::BadRequest(format!(
"logs unavailable for '{name}': log driver '{driver}' cannot be read through the Docker API"
)))
}
}
/// Map a bollard error that is not specific to one container.
fn map_error(e: &BollardError, socket: &str) -> AppError {
match e {
BollardError::DockerResponseServerError {
status_code,
message,
} => match *status_code {
401 | 403 => AppError::Unavailable(format!("docker denied access: {message}")),
404 => AppError::NotFound(message.clone()),
400 => AppError::BadRequest(message.clone()),
501 => AppError::BadRequest(message.clone()),
500 if message.to_ascii_lowercase().contains("logging driver") => {
AppError::BadRequest(message.clone())
}
_ => AppError::Internal(format!("docker API error {status_code}: {message}")),
},
BollardError::IOError { err } => match err.kind() {
ErrorKind::NotFound => {
AppError::Unavailable(format!("docker socket {socket} not found"))
}
ErrorKind::PermissionDenied => {
AppError::Unavailable(format!("permission denied on docker socket {socket}"))
}
ErrorKind::ConnectionRefused => {
AppError::Unavailable(format!("connection refused on docker socket {socket}"))
}
_ => AppError::Unavailable(format!("docker socket {socket}: {err}")),
},
BollardError::SocketNotFoundError(_) => {
AppError::Unavailable(format!("docker socket {socket} not found"))
}
BollardError::RequestTimeoutError => {
AppError::Unavailable(format!("docker socket {socket}: request timed out"))
}
// Transport-level failures mean the daemon is unreachable.
BollardError::HyperResponseError { .. }
| BollardError::HyperLegacyError { .. }
| BollardError::HttpClientError { .. } => {
AppError::Unavailable(format!("docker socket {socket} unreachable: {e}"))
}
other => AppError::Internal(format!("docker: {other}")),
}
}
fn map_container_error(e: &BollardError, socket: &str, name: &str) -> AppError {
match e {
BollardError::DockerResponseServerError {
status_code: 404, ..
} => AppError::NotFound(format!("container '{name}' not found")),
other => map_error(other, socket),
}
}
fn parse_ts(s: &str) -> Option<Timestamp> {
// Never-started containers report year 0001.
s.parse::<Timestamp>().ok().filter(|t| t.as_second() > 0)
}
fn summary_from_inspect(i: &ContainerInspectResponse, now: Timestamp) -> ContainerSummary {
let state = i.state.as_ref();
let state_name = state
.and_then(|s| s.status)
.map_or("unknown", state_str)
.to_owned();
let health = state
.and_then(|s| s.health.as_ref())
.and_then(|h| h.status)
.map(|h| h.to_string())
.filter(|h| !h.is_empty() && h != "none");
let started_at = state
.and_then(|s| s.started_at.as_deref())
.and_then(parse_ts);
let uptime_seconds = match (state_name == "running", started_at) {
(true, Some(t)) => {
Some(u64::try_from((now.as_second() - t.as_second()).max(0)).unwrap_or(0))
}
_ => None,
};
let image = i
.config
.as_ref()
.and_then(|c| c.image.clone())
.filter(|s| !s.is_empty())
.or_else(|| i.image.clone())
.unwrap_or_default();
ContainerSummary {
id: i.id.clone().unwrap_or_default(),
name: i
.name
.as_deref()
.unwrap_or_default()
.trim_start_matches('/')
.to_owned(),
image,
state: state_name,
health,
created: i.created.as_ref().and_then(|c| c.to_string().parse().ok()),
started_at,
uptime_seconds,
restart_count: i.restart_count.and_then(|n| u64::try_from(n).ok()),
}
}
fn status_from_inspect(i: &ContainerInspectResponse, now: Timestamp) -> ContainerStatus {
let state = i.state.as_ref();
let health_output = state
.and_then(|s| s.health.as_ref())
.and_then(|h| h.log.as_ref())
.and_then(|l| l.last())
.and_then(|r| r.output.clone());
ContainerStatus {
summary: summary_from_inspect(i, now),
finished_at: state
.and_then(|s| s.finished_at.as_deref())
.and_then(parse_ts),
exit_code: state.and_then(|s| s.exit_code),
oom_killed: state.and_then(|s| s.oom_killed),
health_output,
image_id: i.image.clone().filter(|s| !s.is_empty()),
resources: i.host_config.as_ref().map(resources_from_host_config),
}
}
/// Copy only limits and sandbox flags. `Env`, `Labels`, `Cmd`, `Args` and mounts
/// stay on the daemon, so no credential can travel through this API.
fn resources_from_host_config(h: &bollard::models::HostConfig) -> ContainerResources {
ContainerResources {
cpu_cores: cpu_cores_from(h.nano_cpus, h.cpu_quota, h.cpu_period),
cpu_shares: h.cpu_shares,
memory_bytes: h.memory.filter(|v| *v > 0),
memory_swap_bytes: h.memory_swap.filter(|v| *v > 0),
pids_limit: h.pids_limit.filter(|v| *v > 0),
privileged: h.privileged,
read_only_rootfs: h.readonly_rootfs,
network_mode: h.network_mode.clone().filter(|s| !s.is_empty()),
restart_policy: h
.restart_policy
.as_ref()
.and_then(|p| p.name)
.map(|n| n.to_string())
.filter(|s| !s.is_empty()),
devices: h
.device_requests
.as_deref()
.unwrap_or_default()
.iter()
.map(device_from_request)
.collect(),
}
}
/// `nano_cpus` wins; a quota/period pair is the older way to say the same thing.
/// Non-positive values mean "unlimited" in both spellings.
fn cpu_cores_from(nano_cpus: Option<i64>, quota: Option<i64>, period: Option<i64>) -> Option<f64> {
if let Some(n) = nano_cpus.filter(|n| *n > 0) {
return Some(n as f64 / 1_000_000_000.0);
}
let (q, p) = (quota.filter(|q| *q > 0)?, period.filter(|p| *p > 0)?);
Some(q as f64 / p as f64)
}
fn device_from_request(r: &bollard::models::DeviceRequest) -> ContainerDevice {
ContainerDevice {
driver: r.driver.clone().filter(|s| !s.is_empty()),
count: r.count,
ids: r.device_ids.clone().unwrap_or_default(),
capabilities: r
.capabilities
.clone()
.unwrap_or_default()
.into_iter()
.flatten()
.collect(),
}
}
fn stats_from_response(
s: &bollard::models::ContainerStatsResponse,
now: Timestamp,
) -> ContainerStats {
let mem = s.memory_stats.as_ref();
let page_cache = mem
.and_then(|m| m.stats.as_ref())
.and_then(|st| st.get("inactive_file").or_else(|| st.get("cache")))
.copied();
let used = mem
.and_then(|m| m.usage)
.map(|u| u.saturating_sub(page_cache.unwrap_or(0)));
let limit = mem.and_then(|m| m.limit).filter(|l| *l > 0);
ContainerStats {
sampled_at: now,
cpu_percent: cpu_percent_from(s),
memory_used_bytes: used,
memory_limit_bytes: limit,
memory_percent: match (used, limit) {
(Some(u), Some(l)) => Some(u as f64 / l as f64 * 100.0),
_ => None,
},
pids: s.pids_stats.as_ref().and_then(|p| p.current),
block_read_bytes: blkio_bytes(s, "read"),
block_write_bytes: blkio_bytes(s, "write"),
network_rx_bytes: network_bytes(s, true),
network_tx_bytes: network_bytes(s, false),
}
}
/// Percent of one core; 400 means four cores saturated. Needs the previous
/// sample the daemon sends alongside, so a zero delta answers `None`.
fn cpu_percent_from(s: &bollard::models::ContainerStatsResponse) -> Option<f64> {
let cpu = s.cpu_stats.as_ref()?;
let pre = s.precpu_stats.as_ref()?;
let total = cpu.cpu_usage.as_ref()?.total_usage?;
let pre_total = pre.cpu_usage.as_ref()?.total_usage?;
let system = cpu.system_cpu_usage?;
let pre_system = pre.system_cpu_usage?;
let cpu_delta = total.checked_sub(pre_total)? as f64;
let system_delta = system.checked_sub(pre_system)? as f64;
if system_delta <= 0.0 {
return None;
}
let cores = cpu
.online_cpus
.filter(|n| *n > 0)
.map(f64::from)
.or_else(|| {
cpu.cpu_usage
.as_ref()
.and_then(|u| u.percpu_usage.as_ref())
.map(|v| v.len() as f64)
})
.unwrap_or(1.0);
Some(cpu_delta / system_delta * cores * 100.0)
}
fn blkio_bytes(s: &bollard::models::ContainerStatsResponse, op: &str) -> Option<u64> {
let entries = s
.blkio_stats
.as_ref()
.and_then(|b| b.io_service_bytes_recursive.as_ref())?;
let total: u64 = entries
.iter()
.filter(|e| e.op.as_deref().is_some_and(|o| o.eq_ignore_ascii_case(op)))
.filter_map(|e| e.value)
.sum();
Some(total)
}
fn network_bytes(s: &bollard::models::ContainerStatsResponse, rx: bool) -> Option<u64> {
let nets = s.networks.as_ref()?;
Some(
nets.values()
.filter_map(|n| if rx { n.rx_bytes } else { n.tx_bytes })
.sum(),
)
}
fn state_str(s: ContainerStateStatusEnum) -> &'static str {
use ContainerStateStatusEnum as S;
match s {
S::EMPTY => "unknown",
S::CREATED => "created",
S::RUNNING => "running",
S::PAUSED => "paused",
S::RESTARTING => "restarting",
S::REMOVING => "removing",
S::EXITED => "exited",
S::DEAD => "dead",
S::STOPPING => "stopping",
}
}
/// Reassembles log frames into lines, one buffer per stream.
#[derive(Default)]
struct LineSplitter {
out: Vec<u8>,
err: Vec<u8>,
}
impl LineSplitter {
fn push(&mut self, frame: LogOutput, emit: &mut dyn FnMut(LogLine)) {
let (stream, buf, bytes) = match frame {
LogOutput::StdErr { message } => (LogStream::Stderr, &mut self.err, message),
LogOutput::StdOut { message } | LogOutput::Console { message } => {
(LogStream::Stdout, &mut self.out, message)
}
LogOutput::StdIn { .. } => return,
};
buf.extend_from_slice(&bytes);
while let Some(pos) = buf.iter().position(|b| *b == b'\n') {
let line: Vec<u8> = buf.drain(..=pos).collect();
emit(parse_line(stream, &line[..line.len() - 1]));
}
}
/// Flush any unterminated trailing lines.
fn finish(&mut self, emit: &mut dyn FnMut(LogLine)) {
for (stream, buf) in [
(LogStream::Stdout, &mut self.out),
(LogStream::Stderr, &mut self.err),
] {
if !buf.is_empty() {
emit(parse_line(stream, buf));
buf.clear();
}
}
}
}
/// Parse `"<RFC3339 ts> <message>"` produced by `timestamps=true`.
fn parse_line(stream: LogStream, raw: &[u8]) -> LogLine {
let text = String::from_utf8_lossy(raw);
let text = text.strip_suffix('\r').unwrap_or(&text);
if let Some((head, rest)) = text.split_once(' ') {
if let Ok(ts) = head.parse::<Timestamp>() {
return LogLine {
stream,
ts: Some(ts),
message: rest.to_owned(),
};
}
}
LogLine {
stream,
ts: None,
message: text.to_owned(),
}
}
#[cfg(test)]
#[path = "docker_tests.rs"]
mod tests;