diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 780f97a..b2920a2 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -81,7 +81,7 @@ Precedence: defaults < config file (`--config`, default `/etc/skyg-linux-tracker - **systemd** (`sources/systemd.rs`): one shared system-bus `zbus::Connection`. `list_units(state)` uses `ListUnits`. `list_timers()` lists `*.timer` units (`ListUnitsByPatterns`) and reads the `Timer` interface on each (`NextElapseUSecRealtime`, `LastTriggerUSec`, `Unit`); `0` or `u64::MAX` means not scheduled. `unit_status(name)` uses `GetUnit` only, never `LoadUnit`, so an unloaded unit is NotFound. `Unit` properties are read for every unit, `Service` properties (`ExecMainStatus`, `NRestarts`) only for `.service` units, so those DTO fields are `Option`. Unit names are validated. No caching. - **journal** (`sources/journal.rs`): all sd-journal access runs inside `spawn_blocking` (a `Journal` is neither `Send` nor `Sync` and is opened and kept in the blocking closure). `query(filter)` seeks backward N entries (unit, priority <= N, `lines` capped by `limits.max_lines`, returned oldest first). `follow(filter)` is a blocking `wait()` loop with a short timeout that feeds an mpsc channel and exits when the receiver is dropped or the shutdown token fires. Binary `MESSAGE` fields are mapped lossily. `sources.journal_namespace` opens a journal namespace. -- **docker** (`sources/docker.rs`): lazily constructed `bollard::Docker` on `sources.docker_socket`. A missing or denied socket never fails startup; each call maps it to `AppError::Unavailable` (HTTP 503 on docker routes only). `ps()` lists containers and inspects them concurrently (bounded) for health, uptime, restart count and image. `logs(name, follow, tail, since)` is a stream that stops on drop or shutdown; `tail` is capped by `limits.max_tail`. +- **docker** (`sources/docker.rs`): lazily constructed `bollard::Docker` on `sources.docker_socket`. A missing or denied socket never fails startup; each call maps it to `AppError::Unavailable` (HTTP 503 on docker routes only). `ps()` lists containers and inspects them concurrently (bounded) for health, uptime, restart count and image. `container_status()` adds the sandbox facts a frozen box otherwise hides: `image_id`, and `resources` (CPU cores, memory and swap limits, pids limit, privileged, read-only rootfs, restart policy, network mode, device requests such as a CDI GPU). Those come from `HostConfig` only — `Env`, `Labels`, `Cmd`, `Args` and mounts are never copied, so no credential can reach the API. `stats(name)` takes one `/stats` sample (`stream=false`, `one-shot=true`) and derives CPU percent from the container/system deltas against `online_cpus`, a working set (usage minus page cache) and IO counters. `logs(name, follow, tail, since)` is a stream that stops on drop or shutdown; `tail` is capped by `limits.max_tail`. ## REST API @@ -96,6 +96,7 @@ All under `/api/v1`, GET only, behind the auth middleware when enabled. Routes u | `GET /api/v1/journal/follow?unit=&priority=` (SSE, event `journal`) | journal `follow()` | | `GET /api/v1/docker/ps` | `ps()` | | `GET /api/v1/docker/containers/{name}` | `container_status()` | +| `GET /api/v1/docker/containers/{name}/stats` | `stats()` — one live sample (CPU, memory, pids, block IO, network) | | `GET /api/v1/docker/containers/{name}/logs?tail=&since=&follow=0\|1` (JSON, or SSE events `stdout`/`stderr`) | `logs()` | | `GET /api/v1/ping` | authenticated liveness, used by a consolidator's API check | | `GET /api/v1/openapi.json` | generated OpenAPI 3.1 document; peer paths and tools only when peers are enabled | diff --git a/TODO.md b/TODO.md index 9fb25d3..7a92c32 100644 --- a/TODO.md +++ b/TODO.md @@ -22,7 +22,7 @@ The build is functionally complete and merged to `main`. REST, MCP, OTel, TLS an | Item | Owner | State | | ---- | ----- | ----------------- | -| — | | Nothing in flight | +| **C-8** container limits and live stats | hermenix | PR open: `feat/container-details` | - [x] **WS-H Gate:** `nix build` and `nix flake check` pass. `ldd` shows no openssl. A fresh NixOS host serves `:1984` and a token can be minted from a login shell. - [x] **V-1** Build-stamped version `YYYY.MM.DD--` in `--version`, `/healthz`, `/api/v1/ping`, MCP server info and OTel `service.version` (DEC-17). `dev-test lint`, `cargo test`, `nix build` and both VM tests (`vm`, `vm peers`) pass. diff --git a/docs/api.md b/docs/api.md index 7eebc6a..8a570b2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -29,7 +29,8 @@ Base: `http://127.0.0.1:1984`. Errors are `{"error":{"code","message"}}`. | `GET /api/v1/journal?unit=&priority=err&lines=100` | priority by name or number | | `GET /api/v1/journal/follow?unit=&priority=` | SSE, event `journal` | | `GET /api/v1/docker/ps` | `503` when Docker is unavailable | -| `GET /api/v1/docker/containers/{name}` | | +| `GET /api/v1/docker/containers/{name}` | limits and sandbox flags (`resources`), `image_id`; never env, labels or commands | +| `GET /api/v1/docker/containers/{name}/stats` | one live sample: CPU percent, memory, pids, block IO, network | | `GET /api/v1/docker/containers/{name}/logs?tail=&since=&follow=0\|1` | JSON, or SSE with events `stdout`/`stderr` | | `GET /api/v1/peers` | consolidator only; sorted status list; absent when peers are disabled | | `GET /api/v1/peers/{peer}` | one peer's status object; unknown name is 404 | @@ -72,7 +73,7 @@ Present only when `[peers] enabled = true`. The same inbound token as the rest o `server` is the TCP check and `api` is the authenticated ping. `auth` is `ok`, `rejected` or `unknown`. Before the first check, `up` is false, `auth` is `unknown` and the timestamps are null. An unknown name is `404` `{"error":{"code":"not_found","message":"peer : unknown peer"}}`. -`GET /api/v1/peers/{peer}/{path}` forwards a GET to that peer and returns the peer's status and body. The path has to be one of the documented read routes: `systemd/units`, `systemd/units/{name}`, `systemd/timers`, `journal`, `journal/follow`, `docker/ps`, `docker/containers/{name}`, `docker/containers/{name}/logs`. `ping` and `peers/...` are not forwarded. `systemd/units/{name}` accepts exactly what the local route accepts, including escaped names (`mnt-data\x2dbackup.mount`, sent as `%5C`) and `-.mount`. `..`, encoded slashes, NUL/CRLF, a second encoding layer and any other path are `400` `peer : path not allowed`. Query keys are the same ones the local route accepts (`state`, `unit`, `priority`, `lines`, `since`, `all`, `tail`, `follow`); anything else is `400` `peer : unsupported query parameter`. The `unit` value is checked with the same unit-name rule as `systemd/units/{name}`, so escaped names work as both `%5C` and a literal backslash (`/api/v1/peers/beta/journal?unit=mnt-data%5Cx2dbackup.mount`). The caller's `Authorization` and `Cookie` are not forwarded; the consolidator sends the peer's own token. +`GET /api/v1/peers/{peer}/{path}` forwards a GET to that peer and returns the peer's status and body. The path has to be one of the documented read routes: `systemd/units`, `systemd/units/{name}`, `systemd/timers`, `journal`, `journal/follow`, `docker/ps`, `docker/containers/{name}`, `docker/containers/{name}/stats`, `docker/containers/{name}/logs`. `ping` and `peers/...` are not forwarded. `systemd/units/{name}` accepts exactly what the local route accepts, including escaped names (`mnt-data\x2dbackup.mount`, sent as `%5C`) and `-.mount`. `..`, encoded slashes, NUL/CRLF, a second encoding layer and any other path are `400` `peer : path not allowed`. Query keys are the same ones the local route accepts (`state`, `unit`, `priority`, `lines`, `since`, `all`, `tail`, `follow`); anything else is `400` `peer : unsupported query parameter`. The `unit` value is checked with the same unit-name rule as `systemd/units/{name}`, so escaped names work as both `%5C` and a literal backslash (`/api/v1/peers/beta/journal?unit=mnt-data%5Cx2dbackup.mount`). The caller's `Authorization` and `Cookie` are not forwarded; the consolidator sends the peer's own token. A request that already has `x-slinuxm-hop` is `400` `peer : chained requests are not allowed`. Other methods are `405`. `journal/follow` and docker logs with `follow=1` are `text/event-stream` with `Cache-Control: no-cache`, like the local follow routes, cut off at `limits.max_follow_secs` or shutdown. Bodies larger than `peers.max_response_bytes` are `503` `peer : oversize`. Only `2xx` from the peer is passed through; any other status becomes the section 4 error for that peer (a peer `404` is `404 not_found`, a peer `401`/`403` is `503 peer rejected credentials`, anything else is `503 http_`), so a redirect or the peer's own error body never reaches the caller. diff --git a/docs/openapi.json b/docs/openapi.json index 2bc522e..3d42bac 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -197,6 +197,79 @@ } } }, + "/api/v1/docker/containers/{name}/stats": { + "get": { + "tags": [ + "docker" + ], + "summary": "One live resource sample of a container", + "description": "A single Docker stats sample: CPU percent, memory, processes, block IO and network counters. Read once per request, so a caller that wants a time series samples this endpoint itself.", + "operationId": "get_container_stats", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "Container name or id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The sample", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ContainerStats" + } + } + } + }, + "400": { + "description": "Invalid container name", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + }, + "401": { + "description": "Missing or unknown bearer token", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + }, + "404": { + "description": "Unknown container", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + }, + "503": { + "description": "Docker socket is not available", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + } + } + } + } + }, "/api/v1/docker/ps": { "get": { "tags": [ @@ -1244,6 +1317,244 @@ "unknown" ] }, + "ContainerDevice": { + "type": "object", + "required": [ + "ids", + "capabilities" + ], + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Capabilities such as `gpu`." + }, + "count": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Devices requested; `None` with `ids` means \"as listed\"." + }, + "driver": { + "type": [ + "string", + "null" + ], + "description": "`nvidia`, `cdi`, …" + }, + "ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Device ids as the driver names them, e.g. `nvidia.com/gpu=all`." + } + } + }, + "ContainerResources": { + "type": "object", + "description": "Limits and sandbox flags taken from the container's `HostConfig`.\n\nDeliberately narrow: `Env`, `Labels`, `Cmd`, `Args` and mounts are never\ncopied, so nothing here can carry a secret.", + "required": [ + "devices" + ], + "properties": { + "cpu_cores": { + "type": [ + "number", + "null" + ], + "format": "double", + "description": "CPU limit in cores (`nano_cpus`, else `cpu_quota`/`cpu_period`). `None` is unlimited." + }, + "cpu_shares": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Relative CPU weight; Docker's default is 1024." + }, + "devices": { + "type": "array", + "items": { + "type": "object", + "required": [ + "ids", + "capabilities" + ], + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Capabilities such as `gpu`." + }, + "count": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Devices requested; `None` with `ids` means \"as listed\"." + }, + "driver": { + "type": [ + "string", + "null" + ], + "description": "`nvidia`, `cdi`, …" + }, + "ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Device ids as the driver names them, e.g. `nvidia.com/gpu=all`." + } + } + }, + "description": "Device reservations, such as a CDI GPU request." + }, + "memory_bytes": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Memory limit in bytes. `None` is unlimited." + }, + "memory_swap_bytes": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Memory plus swap limit in bytes." + }, + "network_mode": { + "type": [ + "string", + "null" + ], + "description": "`bridge`, `host`, `none`, or a compose network name." + }, + "pids_limit": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "privileged": { + "type": [ + "boolean", + "null" + ] + }, + "read_only_rootfs": { + "type": [ + "boolean", + "null" + ] + }, + "restart_policy": { + "type": [ + "string", + "null" + ] + } + } + }, + "ContainerStats": { + "type": "object", + "description": "One live sample from Docker's container stats endpoint.", + "required": [ + "sampled_at" + ], + "properties": { + "block_read_bytes": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "block_write_bytes": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "cpu_percent": { + "type": [ + "number", + "null" + ], + "format": "double", + "description": "CPU use across all cores in percent (100 = one core fully busy)." + }, + "memory_limit_bytes": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "memory_percent": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "memory_used_bytes": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "description": "Working set: usage minus page cache.", + "minimum": 0 + }, + "network_rx_bytes": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "network_tx_bytes": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "pids": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "sampled_at": { + "type": "string", + "format": "date-time" + } + } + }, "ContainerStatus": { "type": "object", "required": [ @@ -1295,6 +1606,13 @@ "image": { "type": "string" }, + "image_id": { + "type": [ + "string", + "null" + ], + "description": "Content address of the image the container runs, e.g. `sha256:…`." + }, "name": { "type": "string", "description": "Name without the leading slash." @@ -1305,6 +1623,124 @@ "null" ] }, + "resources": { + "type": [ + "object", + "null" + ], + "description": "Configured limits and sandbox flags. Never env, labels or command lines.", + "required": [ + "devices" + ], + "properties": { + "cpu_cores": { + "type": [ + "number", + "null" + ], + "format": "double", + "description": "CPU limit in cores (`nano_cpus`, else `cpu_quota`/`cpu_period`). `None` is unlimited." + }, + "cpu_shares": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Relative CPU weight; Docker's default is 1024." + }, + "devices": { + "type": "array", + "items": { + "type": "object", + "required": [ + "ids", + "capabilities" + ], + "properties": { + "capabilities": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Capabilities such as `gpu`." + }, + "count": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Devices requested; `None` with `ids` means \"as listed\"." + }, + "driver": { + "type": [ + "string", + "null" + ], + "description": "`nvidia`, `cdi`, …" + }, + "ids": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Device ids as the driver names them, e.g. `nvidia.com/gpu=all`." + } + } + }, + "description": "Device reservations, such as a CDI GPU request." + }, + "memory_bytes": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Memory limit in bytes. `None` is unlimited." + }, + "memory_swap_bytes": { + "type": [ + "integer", + "null" + ], + "format": "int64", + "description": "Memory plus swap limit in bytes." + }, + "network_mode": { + "type": [ + "string", + "null" + ], + "description": "`bridge`, `host`, `none`, or a compose network name." + }, + "pids_limit": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "privileged": { + "type": [ + "boolean", + "null" + ] + }, + "read_only_rootfs": { + "type": [ + "boolean", + "null" + ] + }, + "restart_policy": { + "type": [ + "string", + "null" + ] + } + } + }, "restart_count": { "type": [ "integer", diff --git a/src/api/behaviour_tests.rs b/src/api/behaviour_tests.rs index f175ff9..0047664 100644 --- a/src/api/behaviour_tests.rs +++ b/src/api/behaviour_tests.rs @@ -24,8 +24,8 @@ use crate::peers::types::{ApiStatus, AuthState, PeerConfig, PeerName, PeerStatus use crate::peers::PeerRegistry; use crate::sources::fake::fake_state; use crate::sources::{ - ContainerStatus, ContainerSummary, DockerSource, JournalEntry, JournalQuery, JournalSource, - LogLine, LogOptions, SystemdSource, TimerInfo, UnitStatus, UnitSummary, + ContainerStats, ContainerStatus, ContainerSummary, DockerSource, JournalEntry, JournalQuery, + JournalSource, LogLine, LogOptions, SystemdSource, TimerInfo, UnitStatus, UnitSummary, }; /// Journal source whose producer sends one entry and then never closes, @@ -208,6 +208,10 @@ impl DockerSource for Strict { ok_name(name)?; Ok(mpsc::channel(1).1) } + async fn stats(&self, name: &str) -> Result { + ok_name(name)?; + Err(AppError::NotFound(name.into())) + } } #[tokio::test] diff --git a/src/api/docker.rs b/src/api/docker.rs index 76f0cc0..6a80daa 100644 --- a/src/api/docker.rs +++ b/src/api/docker.rs @@ -12,7 +12,9 @@ use serde::Deserialize; use super::{query, sse_from}; use crate::error::AppError; use crate::openapi::ErrorBody; -use crate::sources::{ContainerStatus, ContainerSummary, LogLine, LogOptions, LogStream}; +use crate::sources::{ + ContainerStats, ContainerStatus, ContainerSummary, LogLine, LogOptions, LogStream, +}; use crate::state::AppState; pub fn routes() -> Router { @@ -20,6 +22,7 @@ pub fn routes() -> Router { .route("/docker/ps", get(ps)) .route("/docker/containers/{name}", get(container)) .route("/docker/containers/{name}/logs", get(logs)) + .route("/docker/containers/{name}/stats", get(stats)) } #[derive(Deserialize, utoipa::IntoParams)] @@ -89,6 +92,35 @@ pub(crate) async fn container( Ok(Json(st.docker.container_status(&name).await?)) } +#[utoipa::path( + get, + path = "/api/v1/docker/containers/{name}/stats", + operation_id = "get_container_stats", + tag = "docker", + summary = "One live resource sample of a container", + description = "A single Docker stats sample: CPU percent, memory, processes, block IO \ + and network counters. Read once per request, so a caller that wants a \ + time series samples this endpoint itself.", + params(( + "name" = String, + Path, + description = "Container name or id" + )), + responses( + (status = 200, body = ContainerStats, description = "The sample"), + (status = 400, body = ErrorBody, description = "Invalid container name"), + (status = 401, body = ErrorBody, description = "Missing or unknown bearer token"), + (status = 404, body = ErrorBody, description = "Unknown container"), + (status = 503, body = ErrorBody, description = "Docker socket is not available"), + ) +)] +pub(crate) async fn stats( + State(st): State, + Path(name): Path, +) -> Result, AppError> { + Ok(Json(st.docker.stats(&name).await?)) +} + #[derive(Deserialize, utoipa::IntoParams)] #[into_params(parameter_in = Query)] pub(crate) struct LogParams { diff --git a/src/api/docker_tests.rs b/src/api/docker_tests.rs index 48a8470..e81142e 100644 --- a/src/api/docker_tests.rs +++ b/src/api/docker_tests.rs @@ -9,7 +9,9 @@ use crate::api::testutil::{get, json}; use crate::config::Config; use crate::error::AppError; use crate::sources::fake::fake_state; -use crate::sources::{ContainerStatus, ContainerSummary, DockerSource, LogLine, LogOptions}; +use crate::sources::{ + ContainerStats, ContainerStatus, ContainerSummary, DockerSource, LogLine, LogOptions, +}; fn app() -> axum::Router { router(fake_state(Config::default())) @@ -24,7 +26,27 @@ async fn ps_and_detail() { assert_eq!(json(&b).as_array().unwrap().len(), 3); let (s, b) = get(&app(), "/docker/containers/web").await; assert_eq!(s, StatusCode::OK); - assert_eq!(json(&b)["health"], "healthy"); + let detail = json(&b); + assert_eq!(detail["health"], "healthy"); + // Limits and image identity are part of the detail view. + assert_eq!(detail["image_id"], "sha256:aaaa1111"); + assert_eq!(detail["resources"]["cpu_cores"], 2.0); + assert_eq!(detail["resources"]["memory_bytes"], 2147483648i64); + assert_eq!(detail["resources"]["restart_policy"], "unless-stopped"); +} + +#[tokio::test] +async fn stats_sample_and_unknown_container() { + let (s, b) = get(&app(), "/docker/containers/web/stats").await; + assert_eq!(s, StatusCode::OK); + let v = json(&b); + assert_eq!(v["cpu_percent"], 12.5); + assert_eq!(v["pids"], 7); + assert_eq!(v["memory_limit_bytes"], 2147483648i64); + assert_eq!(v["network_rx_bytes"], 1024); + let (s, b) = get(&app(), "/docker/containers/nope/stats").await; + assert_eq!(s, StatusCode::NOT_FOUND); + assert_eq!(json(&b)["error"]["code"], "not_found"); } #[tokio::test] @@ -90,6 +112,9 @@ impl DockerSource for Down { ) -> Result, AppError> { Err(AppError::Unavailable("docker socket".into())) } + async fn stats(&self, _: &str) -> Result { + Err(AppError::Unavailable("docker socket".into())) + } } #[tokio::test] @@ -101,6 +126,7 @@ async fn docker_unavailable_isolated() { "/docker/ps", "/docker/containers/web", "/docker/containers/web/logs", + "/docker/containers/web/stats", "/docker/containers/web/logs?follow=1", ] { let (s, b) = get(&app, u).await; diff --git a/src/openapi.rs b/src/openapi.rs index 1a77b45..f87814c 100644 --- a/src/openapi.rs +++ b/src/openapi.rs @@ -34,7 +34,8 @@ use utoipa::{OpenApi as OpenApiDoc, PartialSchema, ToSchema}; use crate::mcp; use crate::peers::types::{ApiStatus, AuthState, PeerStatus, ServerStatus}; use crate::sources::{ - ContainerStatus, ContainerSummary, JournalEntry, LogLine, TimerInfo, UnitStatus, UnitSummary, + ContainerDevice, ContainerResources, ContainerStats, ContainerStatus, ContainerSummary, + JournalEntry, LogLine, TimerInfo, UnitStatus, UnitSummary, }; /// Route prefix of the peer surface. Those routes are merged only when a peer @@ -57,6 +58,7 @@ pub const BEARER_SCHEME: &str = "bearer_auth"; crate::api::docker::ps, crate::api::docker::container, crate::api::docker::logs, + crate::api::docker::stats, crate::api::peers::list, crate::api::peers::one, crate::api::peers::pass, @@ -75,6 +77,9 @@ pub const BEARER_SCHEME: &str = "bearer_auth"; JournalEntry, ContainerSummary, ContainerStatus, + ContainerResources, + ContainerDevice, + ContainerStats, LogLine, PeerStatus, ServerStatus, @@ -357,6 +362,7 @@ macro_rules! reflected_schema { reflected_schema!(UnitSummary, UnitStatus, TimerInfo, JournalEntry); reflected_schema!(ContainerSummary, ContainerStatus, LogLine); +reflected_schema!(ContainerResources, ContainerDevice, ContainerStats); reflected_schema!(PeerStatus, ServerStatus, ApiStatus, AuthState); /// Translate one JSON Schema 2020-12 object into utoipa's model. diff --git a/src/openapi_tests.rs b/src/openapi_tests.rs index 254709b..dde1e9c 100644 --- a/src/openapi_tests.rs +++ b/src/openapi_tests.rs @@ -28,6 +28,9 @@ fn reflected_schemas_match_schemars() { check!( ApiStatus, AuthState, + ContainerDevice, + ContainerResources, + ContainerStats, ContainerStatus, ContainerSummary, JournalEntry, diff --git a/src/peers/proxy.rs b/src/peers/proxy.rs index 24c469a..e4c46df 100644 --- a/src/peers/proxy.rs +++ b/src/peers/proxy.rs @@ -330,10 +330,11 @@ fn container_path(route: &str) -> bool { let Some(rest) = route.strip_prefix("docker/containers/") else { return false; }; - match rest.strip_suffix("/logs") { - Some(name) => resource_segment(name), - None => resource_segment(rest), - } + let name = rest + .strip_suffix("/logs") + .or_else(|| rest.strip_suffix("/stats")) + .unwrap_or(rest); + resource_segment(name) } fn resource_segment(name: &str) -> bool { diff --git a/src/peers/proxy_tests.rs b/src/peers/proxy_tests.rs index 0121f33..87f5cb2 100644 --- a/src/peers/proxy_tests.rs +++ b/src/peers/proxy_tests.rs @@ -90,6 +90,7 @@ async fn allowlist_forwards_documented_routes_only() { "/docker/ps", "/docker/containers/web", "/docker/containers/web/logs", + "/docker/containers/web/stats", "/api/v1/systemd/units", "systemd/units/user@100.service", // Names the local route accepts must pass here too: `-` roots and @@ -118,6 +119,7 @@ async fn allowlist_forwards_documented_routes_only() { "/api/v1/docker/ps", "/api/v1/docker/containers/web", "/api/v1/docker/containers/web/logs", + "/api/v1/docker/containers/web/stats", "/api/v1/systemd/units", "/api/v1/systemd/units/user@100.service", "/api/v1/systemd/units/-.mount", diff --git a/src/sources/docker.rs b/src/sources/docker.rs index b758b4b..a3a5791 100644 --- a/src/sources/docker.rs +++ b/src/sources/docker.rs @@ -14,6 +14,7 @@ use bollard::errors::Error as BollardError; use bollard::models::{ContainerInspectResponse, ContainerStateStatusEnum}; use bollard::query_parameters::{ InspectContainerOptions, ListContainersOptionsBuilder, LogsOptions, LogsOptionsBuilder, + StatsOptionsBuilder, }; use bollard::{Docker, API_DEFAULT_VERSION}; use futures::stream::{self, StreamExt}; @@ -22,7 +23,8 @@ use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; use super::{ - clamp_count, ContainerStatus, ContainerSummary, DockerSource, LogLine, LogOptions, LogStream, + clamp_count, ContainerDevice, ContainerResources, ContainerStats, ContainerStatus, + ContainerSummary, DockerSource, LogLine, LogOptions, LogStream, }; use crate::config::Config; use crate::error::AppError; @@ -215,6 +217,28 @@ impl DockerSource for BollardDocker { }); Ok(rx) } + + async fn stats(&self, name: &str) -> Result { + let name = normalize_name(name)?; + let docker = self.client()?; + let socket = self.socket_name(); + // One shot: the daemon samples once, answers and closes the stream. + let opts = StatsOptionsBuilder::new() + .stream(false) + .one_shot(true) + .build(); + let mut s = Box::pin(docker.stats(&name, Some(opts))); + let sample = match s.next().await { + Some(Ok(sample)) => sample, + Some(Err(e)) => return Err(map_container_error(&e, &socket, &name)), + None => { + return Err(AppError::Internal(format!( + "docker returned no stats for '{name}'" + ))) + } + }; + Ok(stats_from_response(&sample, Timestamp::now())) + } } /// Strip the leading `/` and reject names that could alter the request path. @@ -379,9 +403,144 @@ fn status_from_inspect(i: &ContainerInspectResponse, now: Timestamp) -> Containe exit_code: state.and_then(|s| s.exit_code), oom_killed: state.and_then(|s| s.oom_killed), health_output, + image_id: i.image.clone().filter(|s| !s.is_empty()), + resources: i.host_config.as_ref().map(resources_from_host_config), } } +/// Copy only limits and sandbox flags. `Env`, `Labels`, `Cmd`, `Args` and mounts +/// stay on the daemon, so no credential can travel through this API. +fn resources_from_host_config(h: &bollard::models::HostConfig) -> ContainerResources { + ContainerResources { + cpu_cores: cpu_cores_from(h.nano_cpus, h.cpu_quota, h.cpu_period), + cpu_shares: h.cpu_shares, + memory_bytes: h.memory.filter(|v| *v > 0), + memory_swap_bytes: h.memory_swap.filter(|v| *v > 0), + pids_limit: h.pids_limit.filter(|v| *v > 0), + privileged: h.privileged, + read_only_rootfs: h.readonly_rootfs, + network_mode: h.network_mode.clone().filter(|s| !s.is_empty()), + restart_policy: h + .restart_policy + .as_ref() + .and_then(|p| p.name) + .map(|n| n.to_string()) + .filter(|s| !s.is_empty()), + devices: h + .device_requests + .as_deref() + .unwrap_or_default() + .iter() + .map(device_from_request) + .collect(), + } +} + +/// `nano_cpus` wins; a quota/period pair is the older way to say the same thing. +/// Non-positive values mean "unlimited" in both spellings. +fn cpu_cores_from(nano_cpus: Option, quota: Option, period: Option) -> Option { + if let Some(n) = nano_cpus.filter(|n| *n > 0) { + return Some(n as f64 / 1_000_000_000.0); + } + let (q, p) = (quota.filter(|q| *q > 0)?, period.filter(|p| *p > 0)?); + Some(q as f64 / p as f64) +} + +fn device_from_request(r: &bollard::models::DeviceRequest) -> ContainerDevice { + ContainerDevice { + driver: r.driver.clone().filter(|s| !s.is_empty()), + count: r.count, + ids: r.device_ids.clone().unwrap_or_default(), + capabilities: r + .capabilities + .clone() + .unwrap_or_default() + .into_iter() + .flatten() + .collect(), + } +} + +fn stats_from_response( + s: &bollard::models::ContainerStatsResponse, + now: Timestamp, +) -> ContainerStats { + let mem = s.memory_stats.as_ref(); + let page_cache = mem + .and_then(|m| m.stats.as_ref()) + .and_then(|st| st.get("inactive_file").or_else(|| st.get("cache"))) + .copied(); + let used = mem + .and_then(|m| m.usage) + .map(|u| u.saturating_sub(page_cache.unwrap_or(0))); + let limit = mem.and_then(|m| m.limit).filter(|l| *l > 0); + ContainerStats { + sampled_at: now, + cpu_percent: cpu_percent_from(s), + memory_used_bytes: used, + memory_limit_bytes: limit, + memory_percent: match (used, limit) { + (Some(u), Some(l)) => Some(u as f64 / l as f64 * 100.0), + _ => None, + }, + pids: s.pids_stats.as_ref().and_then(|p| p.current), + block_read_bytes: blkio_bytes(s, "read"), + block_write_bytes: blkio_bytes(s, "write"), + network_rx_bytes: network_bytes(s, true), + network_tx_bytes: network_bytes(s, false), + } +} + +/// Percent of one core; 400 means four cores saturated. Needs the previous +/// sample the daemon sends alongside, so a zero delta answers `None`. +fn cpu_percent_from(s: &bollard::models::ContainerStatsResponse) -> Option { + let cpu = s.cpu_stats.as_ref()?; + let pre = s.precpu_stats.as_ref()?; + let total = cpu.cpu_usage.as_ref()?.total_usage?; + let pre_total = pre.cpu_usage.as_ref()?.total_usage?; + let system = cpu.system_cpu_usage?; + let pre_system = pre.system_cpu_usage?; + let cpu_delta = total.checked_sub(pre_total)? as f64; + let system_delta = system.checked_sub(pre_system)? as f64; + if system_delta <= 0.0 { + return None; + } + let cores = cpu + .online_cpus + .filter(|n| *n > 0) + .map(f64::from) + .or_else(|| { + cpu.cpu_usage + .as_ref() + .and_then(|u| u.percpu_usage.as_ref()) + .map(|v| v.len() as f64) + }) + .unwrap_or(1.0); + Some(cpu_delta / system_delta * cores * 100.0) +} + +fn blkio_bytes(s: &bollard::models::ContainerStatsResponse, op: &str) -> Option { + let entries = s + .blkio_stats + .as_ref() + .and_then(|b| b.io_service_bytes_recursive.as_ref())?; + let total: u64 = entries + .iter() + .filter(|e| e.op.as_deref().is_some_and(|o| o.eq_ignore_ascii_case(op))) + .filter_map(|e| e.value) + .sum(); + Some(total) +} + +fn network_bytes(s: &bollard::models::ContainerStatsResponse, rx: bool) -> Option { + let nets = s.networks.as_ref()?; + Some( + nets.values() + .filter_map(|n| if rx { n.rx_bytes } else { n.tx_bytes }) + .sum(), + ) +} + fn state_str(s: ContainerStateStatusEnum) -> &'static str { use ContainerStateStatusEnum as S; match s { diff --git a/src/sources/docker_tests.rs b/src/sources/docker_tests.rs index 244dc2d..18ace27 100644 --- a/src/sources/docker_tests.rs +++ b/src/sources/docker_tests.rs @@ -120,4 +120,173 @@ async fn missing_socket_is_unavailable() { d.follow_logs("x", &LogOptions::default()).await, Err(AppError::Unavailable(_)) )); + assert!(matches!(d.stats("x").await, Err(AppError::Unavailable(_)))); +} + +#[test] +fn limits_become_plain_numbers() { + use bollard::models::{DeviceRequest, HostConfig, RestartPolicy, RestartPolicyNameEnum}; + let h = HostConfig { + nano_cpus: Some(4_000_000_000), + cpu_shares: Some(1024), + memory: Some(16 * 1024 * 1024 * 1024), + memory_swap: Some(-1), + pids_limit: Some(0), + privileged: Some(false), + readonly_rootfs: Some(true), + network_mode: Some("bridge".into()), + restart_policy: Some(RestartPolicy { + name: Some(RestartPolicyNameEnum::ALWAYS), + maximum_retry_count: Some(0), + }), + device_requests: Some(vec![DeviceRequest { + driver: Some("cdi".into()), + count: None, + device_ids: Some(vec!["nvidia.com/gpu=all".into()]), + capabilities: Some(vec![vec!["gpu".into()]]), + options: None, + }]), + ..Default::default() + }; + let r = resources_from_host_config(&h); + assert_eq!(r.cpu_cores, Some(4.0)); + assert_eq!(r.cpu_shares, Some(1024)); + assert_eq!(r.memory_bytes, Some(16 * 1024 * 1024 * 1024)); + // -1 and 0 both mean "unlimited", so neither is reported as a limit. + assert_eq!(r.memory_swap_bytes, None); + assert_eq!(r.pids_limit, None); + assert_eq!(r.read_only_rootfs, Some(true)); + assert_eq!(r.network_mode.as_deref(), Some("bridge")); + assert_eq!(r.restart_policy.as_deref(), Some("always")); + assert_eq!(r.devices.len(), 1); + assert_eq!(r.devices[0].driver.as_deref(), Some("cdi")); + assert_eq!(r.devices[0].ids, ["nvidia.com/gpu=all"]); + assert_eq!(r.devices[0].capabilities, ["gpu"]); +} + +#[test] +fn quota_and_period_are_cores_too() { + assert_eq!( + cpu_cores_from(None, Some(150_000), Some(100_000)), + Some(1.5) + ); + assert_eq!(cpu_cores_from(None, Some(-1), Some(100_000)), None); + assert_eq!(cpu_cores_from(None, Some(50_000), None), None); + assert_eq!( + cpu_cores_from(Some(500_000_000), Some(150_000), Some(100_000)), + Some(0.5) + ); +} + +#[test] +fn details_carry_no_env_labels_or_commands() { + use bollard::models::{ContainerConfig, HostConfig}; + let i = ContainerInspectResponse { + id: Some("abc".into()), + image: Some("sha256:deadbeef".into()), + config: Some(ContainerConfig { + env: Some(vec!["API_TOKEN=supersecret".into()]), + cmd: Some(vec!["--password=letmein".into()]), + labels: Some(std::collections::HashMap::from([( + "compose.secret".to_string(), + "hunter2".to_string(), + )])), + ..Default::default() + }), + host_config: Some(HostConfig { + nano_cpus: Some(1_000_000_000), + ..Default::default() + }), + ..Default::default() + }; + let now: Timestamp = "2026-10-15T12:01:00Z".parse().unwrap(); + let json = serde_json::to_string(&status_from_inspect(&i, now)).unwrap(); + for leaked in [ + "supersecret", + "letmein", + "hunter2", + "\"env\"", + "\"labels\"", + "\"cmd\"", + ] { + assert!(!json.contains(leaked), "{leaked} leaked into {json}"); + } + assert!(json.contains("sha256:deadbeef")); + assert!(json.contains("\"cpu_cores\":1.0")); +} + +#[test] +fn stats_are_derived_from_deltas() { + use bollard::models::{ + ContainerBlkioStatEntry, ContainerBlkioStats, ContainerCpuStats, ContainerCpuUsage, + ContainerMemoryStats, ContainerNetworkStats, ContainerPidsStats, ContainerStatsResponse, + }; + let s = ContainerStatsResponse { + cpu_stats: Some(ContainerCpuStats { + cpu_usage: Some(ContainerCpuUsage { + total_usage: Some(3_000_000_000), + ..Default::default() + }), + system_cpu_usage: Some(10_000_000_000), + online_cpus: Some(4), + ..Default::default() + }), + precpu_stats: Some(ContainerCpuStats { + cpu_usage: Some(ContainerCpuUsage { + total_usage: Some(2_000_000_000), + ..Default::default() + }), + system_cpu_usage: Some(8_000_000_000), + ..Default::default() + }), + memory_stats: Some(ContainerMemoryStats { + usage: Some(600 * 1024 * 1024), + limit: Some(1024 * 1024 * 1024), + stats: Some(std::collections::HashMap::from([( + "inactive_file".to_string(), + 100 * 1024 * 1024, + )])), + ..Default::default() + }), + pids_stats: Some(ContainerPidsStats { + current: Some(12), + ..Default::default() + }), + blkio_stats: Some(ContainerBlkioStats { + io_service_bytes_recursive: Some(vec![ + ContainerBlkioStatEntry { + op: Some("Read".into()), + value: Some(100), + ..Default::default() + }, + ContainerBlkioStatEntry { + op: Some("Write".into()), + value: Some(200), + ..Default::default() + }, + ]), + ..Default::default() + }), + networks: Some(std::collections::HashMap::from([( + "eth0".to_string(), + ContainerNetworkStats { + rx_bytes: Some(1000), + tx_bytes: Some(2000), + ..Default::default() + }, + )])), + ..Default::default() + }; + let now: Timestamp = "2026-10-15T12:01:00Z".parse().unwrap(); + let st = stats_from_response(&s, now); + // (3-2)GiB of container CPU over (10-8)GiB of system CPU, 4 cores. + assert!((st.cpu_percent.unwrap() - 200.0).abs() < 1e-9); + // Page cache is not the container's working set. + assert_eq!(st.memory_used_bytes, Some(500 * 1024 * 1024)); + assert!((st.memory_percent.unwrap() - 48.828_125).abs() < 1e-6); + assert_eq!(st.pids, Some(12)); + assert_eq!(st.block_read_bytes, Some(100)); + assert_eq!(st.block_write_bytes, Some(200)); + assert_eq!(st.network_rx_bytes, Some(1000)); + assert_eq!(st.network_tx_bytes, Some(2000)); } diff --git a/src/sources/fake.rs b/src/sources/fake.rs index b7dc085..ea6d9ea 100644 --- a/src/sources/fake.rs +++ b/src/sources/fake.rs @@ -11,9 +11,9 @@ use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; use super::{ - clamp_count, ContainerStatus, ContainerSummary, DockerSource, JournalEntry, JournalQuery, - JournalSource, LogLine, LogOptions, LogStream, Priority, SystemdSource, TimerInfo, UnitStatus, - UnitSummary, + clamp_count, ContainerResources, ContainerStats, ContainerStatus, ContainerSummary, + DockerSource, JournalEntry, JournalQuery, JournalSource, LogLine, LogOptions, LogStream, + Priority, SystemdSource, TimerInfo, UnitStatus, UnitSummary, }; use crate::config::Config; use crate::error::AppError; @@ -239,6 +239,19 @@ impl FakeDocker { exit_code: None, oom_killed: Some(false), health_output: health.map(|h| format!("probe says {h}")), + image_id: Some(format!("sha256:{id}")), + resources: Some(ContainerResources { + cpu_cores: Some(2.0), + cpu_shares: Some(1024), + memory_bytes: Some(2 * 1024 * 1024 * 1024), + memory_swap_bytes: None, + pids_limit: Some(512), + privileged: Some(false), + read_only_rootfs: Some(true), + network_mode: Some("bridge".into()), + restart_policy: Some("unless-stopped".into()), + devices: vec![], + }), }; Self { containers: vec![ @@ -285,6 +298,22 @@ impl DockerSource for FakeDocker { self.find(name).cloned() } + async fn stats(&self, name: &str) -> Result { + self.find(name)?; + Ok(ContainerStats { + sampled_at: ts("2026-10-15T09:05:00Z"), + cpu_percent: Some(12.5), + memory_used_bytes: Some(256 * 1024 * 1024), + memory_limit_bytes: Some(2 * 1024 * 1024 * 1024), + memory_percent: Some(12.5), + pids: Some(7), + block_read_bytes: Some(4096), + block_write_bytes: Some(8192), + network_rx_bytes: Some(1024), + network_tx_bytes: Some(2048), + }) + } + async fn logs(&self, name: &str, opts: &LogOptions) -> Result, AppError> { self.find(name)?; let n = clamp_count(opts.tail, usize::MAX); diff --git a/src/sources/mod.rs b/src/sources/mod.rs index 0631c45..9ad5ab1 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -275,6 +275,63 @@ pub struct ContainerStatus { pub oom_killed: Option, /// Last health check output, if any. pub health_output: Option, + /// Content address of the image the container runs, e.g. `sha256:…`. + pub image_id: Option, + /// Configured limits and sandbox flags. Never env, labels or command lines. + pub resources: Option, +} + +/// Limits and sandbox flags taken from the container's `HostConfig`. +/// +/// Deliberately narrow: `Env`, `Labels`, `Cmd`, `Args` and mounts are never +/// copied, so nothing here can carry a secret. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ContainerResources { + /// CPU limit in cores (`nano_cpus`, else `cpu_quota`/`cpu_period`). `None` is unlimited. + pub cpu_cores: Option, + /// Relative CPU weight; Docker's default is 1024. + pub cpu_shares: Option, + /// Memory limit in bytes. `None` is unlimited. + pub memory_bytes: Option, + /// Memory plus swap limit in bytes. + pub memory_swap_bytes: Option, + pub pids_limit: Option, + pub privileged: Option, + pub read_only_rootfs: Option, + /// `bridge`, `host`, `none`, or a compose network name. + pub network_mode: Option, + pub restart_policy: Option, + /// Device reservations, such as a CDI GPU request. + pub devices: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ContainerDevice { + /// `nvidia`, `cdi`, … + pub driver: Option, + /// Devices requested; `None` with `ids` means "as listed". + pub count: Option, + /// Device ids as the driver names them, e.g. `nvidia.com/gpu=all`. + pub ids: Vec, + /// Capabilities such as `gpu`. + pub capabilities: Vec, +} + +/// One live sample from Docker's container stats endpoint. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ContainerStats { + pub sampled_at: Timestamp, + /// CPU use across all cores in percent (100 = one core fully busy). + pub cpu_percent: Option, + /// Working set: usage minus page cache. + pub memory_used_bytes: Option, + pub memory_limit_bytes: Option, + pub memory_percent: Option, + pub pids: Option, + pub block_read_bytes: Option, + pub block_write_bytes: Option, + pub network_rx_bytes: Option, + pub network_tx_bytes: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] @@ -315,6 +372,8 @@ pub trait DockerSource: Send + Sync { name: &str, opts: &LogOptions, ) -> Result, AppError>; + /// One live resource sample (CPU, memory, IO). Costs a daemon round trip. + async fn stats(&self, name: &str) -> Result; } /// Default value of `JournalQuery.lines` and `LogOptions.tail`. diff --git a/tests/openapi.rs b/tests/openapi.rs index 228c6b8..37f3b14 100644 --- a/tests/openapi.rs +++ b/tests/openapi.rs @@ -332,7 +332,7 @@ async fn every_router_route_is_documented() { let mut routes = api_routes(); routes.extend(server_routes()); // A scan that stops matching would make the checks below vacuous. - assert_eq!(routes.len(), 17, "route scan found: {routes:?}"); + assert_eq!(routes.len(), 18, "route scan found: {routes:?}"); for route in routes { let path = openapi_path(&route.path); let item =