This commit is contained in:
ari committed 2026-04-15 12:57:46 -04:00
1 parent 0c8f136b14
commit 92df288796
8 files changed
+400 -4

No files matched your search

+8 -4
View File
@@ -42,13 +42,17 @@ in
niri.enable = true;
};
};
server.services.comfyui = {
enable = true;
mode = "rocm";
port = ports.comfyui;
rocm = {
dataDir = "/data/comfyui";
};
};
};
networkDrives = {
enable = true;
# options = [
# "x-systemd.automount"
# "noauto"
# ];
};
};
users.users.ari.extraGroups = [
@@ -0,0 +1,23 @@
{ lib, config, pkgs, ... }:
let
cfg = config.skyg.nixos.server.services.comfyui;
isCuda = cfg.mode == "cuda";
in
{
config = lib.mkIf (cfg.enable && isCuda) {
# TODO: Add CUDA-specific assertions
# assertions = [
# {
# assertion = config.hardware.nvidia.enable or false;
# message = "ComfyUI CUDA mode requires NVIDIA drivers to be enabled";
# }
# ];
# TODO: Add CUDA-specific packages
# environment.systemPackages = with pkgs; [
# cudaPackages.cudatoolkit
# nvidia-docker
# ];
};
}
@@ -0,0 +1,9 @@
{ ... }:
{
imports = [
./shared.nix
./rocm.nix
./rocm-docker.nix
./cuda.nix
];
}
@@ -0,0 +1,174 @@
# ComfyUI ROCm Docker Container Build and Service
#
# This module builds the ComfyUI container image and runs it as a systemd service
# using NixOS's OCI container abstraction.
{ lib, config, pkgs, ... }:
let
cfg = config.skyg.nixos.server.services.comfyui;
isRocm = cfg.enable && cfg.mode == "rocm";
# Load Containerfile from file
containerfile = ./rocm.Containerfile;
# Image name and tag
imageName = "comfyui-rocm";
imageTag = cfg.rocm.imageTag;
fullImageName = "${imageName}:${imageTag}";
in
{
options.skyg.nixos.server.services.comfyui.rocm = with lib; {
uid = mkOption {
type = types.int;
default = 1000;
description = "User ID to run the ComfyUI container as";
};
gid = mkOption {
type = types.int;
default = 100;
description = "Group ID to run the ComfyUI container as";
};
hsaOverrideGfxVersion = mkOption {
type = types.str;
default = "11.5.1";
description = "HSA_OVERRIDE_GFX_VERSION for ROCm compatibility";
};
comfyuiCommit = mkOption {
type = types.str;
default = "master";
description = "ComfyUI git commit or branch to checkout";
};
imageTag = mkOption {
type = types.str;
default = "latest";
description = "Tag for the built ComfyUI ROCm image";
};
dataDir = mkOption {
type = types.path;
default = "/var/lib/comfyui";
description = "Directory for ComfyUI data (models, output, etc.)";
};
modelsDir = mkOption {
type = types.nullOr types.path;
default = null;
example = "/mnt/models";
description = "Optional external models directory to mount";
};
outputDir = mkOption {
type = types.nullOr types.path;
default = null;
example = "/mnt/output";
description = "Optional external output directory to mount";
};
extraVolumes = mkOption {
type = types.listOf types.str;
default = [ ];
example = [ "/path/to/custom:/workspace/ComfyUI/custom_nodes/custom" ];
description = "Additional volume mounts for the container";
};
autoStart = mkOption {
type = types.bool;
default = true;
description = "Whether to start the ComfyUI container automatically";
};
rebuildImage = mkOption {
type = types.bool;
default = false;
description = "Force rebuild the Docker image on next activation";
};
};
config = lib.mkIf isRocm {
# Create data directories with proper ownership
systemd.tmpfiles.rules = [
"d ${cfg.rocm.dataDir} 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/checkpoints 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/vae 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/loras 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/unet 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/clip 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/controlnet 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/output 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/custom_nodes 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
];
# Build script for the Docker image
system.activationScripts.comfyui-rocm-build = {
text = ''
# Check if image exists or rebuild is requested
if ! ${pkgs.docker}/bin/docker image inspect ${fullImageName} > /dev/null 2>&1 || ${lib.boolToString cfg.rocm.rebuildImage}; then
echo "Building ComfyUI ROCm Docker image..."
${pkgs.docker}/bin/docker build \
-t ${fullImageName} \
--build-arg COMFYUI_COMMIT=${cfg.rocm.comfyuiCommit} \
--build-arg HSA_OVERRIDE_GFX_VERSION=${cfg.rocm.hsaOverrideGfxVersion} \
-f ${containerfile} \
${pkgs.emptyDirectory}
else
echo "ComfyUI ROCm Docker image already exists, skipping build."
fi
'';
deps = [ ];
};
# OCI container definition
virtualisation.oci-containers.containers.comfyui = {
autoStart = cfg.rocm.autoStart;
image = fullImageName;
ports = [
"${toString cfg.port}:8188"
];
# Run as user and ROCm GPU access
user = "${toString cfg.rocm.uid}:${toString cfg.rocm.gid}";
extraOptions = [
"--device=/dev/kfd"
"--device=/dev/dri"
"--group-add=video"
"--group-add=render"
"--security-opt=seccomp=unconfined"
"--ipc=host"
];
environment = {
HSA_OVERRIDE_GFX_VERSION = cfg.rocm.hsaOverrideGfxVersion;
HSA_ENABLE_SDMA = "0";
ROCBLAS_USE_HIPBLASLT = "1";
PYTORCH_HIP_ALLOC_CONF = "expandable_segments:False";
};
volumes = [
"${cfg.rocm.dataDir}/custom_nodes:/workspace/ComfyUI/custom_nodes"
] ++ (if cfg.rocm.modelsDir != null
then [ "${cfg.rocm.modelsDir}:/workspace/ComfyUI/models" ]
else [ "${cfg.rocm.dataDir}/models:/workspace/ComfyUI/models" ])
++ (if cfg.rocm.outputDir != null
then [ "${cfg.rocm.outputDir}:/workspace/ComfyUI/output" ]
else [ "${cfg.rocm.dataDir}/output:/workspace/ComfyUI/output" ])
++ cfg.rocm.extraVolumes;
};
# Ensure the container service starts after Docker image is built
systemd.services."docker-comfyui" = {
after = [ "docker.service" ];
requires = [ "docker.service" ];
serviceConfig = {
Restart = "on-failure";
RestartSec = "10s";
};
};
};
}
@@ -0,0 +1,58 @@
FROM rocm/pytorch:rocm7.2.1_ubuntu24.04_py3.12_pytorch_release_2.9.1
# ─── ROCm environment for gfx1151 (Strix Halo) ───────────────────────────
ENV HSA_OVERRIDE_GFX_VERSION=11.5.1
ENV HSA_ENABLE_SDMA=0
ENV ROCBLAS_USE_HIPBLASLT=1
ENV PYTORCH_HIP_ALLOC_CONF=expandable_segments:False
# ─── Clone ComfyUI ────────────────────────────────────────────────────────
ARG COMFYUI_COMMIT=master
ENV COMFYUI_PATH=/workspace/ComfyUI
RUN git clone https://github.com/comfyanonymous/ComfyUI "$COMFYUI_PATH" \
&& cd "$COMFYUI_PATH" \
&& if [ "$COMFYUI_COMMIT" != "master" ]; then git checkout "$COMFYUI_COMMIT"; fi
# ─── Install ComfyUI requirements ─────────────────────────────────────────
RUN pip install --no-cache-dir -r "$COMFYUI_PATH/requirements.txt"
# ─── Custom nodes ─────────────────────────────────────────────────────────
# ComfyUI-Manager
RUN git clone https://github.com/ltdrdata/ComfyUI-Manager \
"$COMFYUI_PATH/custom_nodes/ComfyUI-Manager" \
&& pip install --no-cache-dir \
-r "$COMFYUI_PATH/custom_nodes/ComfyUI-Manager/requirements.txt"
# WAN 2.1 video nodes
RUN git clone https://github.com/kijai/ComfyUI-WanVideoWrapper \
"$COMFYUI_PATH/custom_nodes/ComfyUI-WanVideoWrapper" \
&& pip install --no-cache-dir \
-r "$COMFYUI_PATH/custom_nodes/ComfyUI-WanVideoWrapper/requirements.txt" \
|| true
# Video Helper Suite
RUN git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite \
"$COMFYUI_PATH/custom_nodes/ComfyUI-VideoHelperSuite" \
&& pip install --no-cache-dir \
-r "$COMFYUI_PATH/custom_nodes/ComfyUI-VideoHelperSuite/requirements.txt" \
|| true
# ─── Model directories ────────────────────────────────────────────────────
RUN mkdir -p \
"$COMFYUI_PATH/models/checkpoints" \
"$COMFYUI_PATH/models/vae" \
"$COMFYUI_PATH/models/loras" \
"$COMFYUI_PATH/models/unet" \
"$COMFYUI_PATH/models/clip" \
"$COMFYUI_PATH/models/controlnet" \
"$COMFYUI_PATH/output"
WORKDIR "$COMFYUI_PATH"
EXPOSE 8188
CMD ["python", "main.py", \
"--listen", "0.0.0.0", \
"--port", "8188", \
"--use-pytorch-cross-attention"]
@@ -0,0 +1,58 @@
{ lib, config, pkgs, ... }:
let
cfg = config.skyg.nixos.server.services.comfyui;
isRocm = cfg.enable && cfg.mode == "rocm";
in
{
options.skyg.nixos.server.services.comfyui = with lib; {
rocmOverrideGfx = mkOption {
type = types.nullOr types.str;
default = null;
example = "gfx1151";
description = "Override GFX version for ROCm (useful for newer AMD GPUs)";
};
};
config = lib.mkIf isRocm {
# ROCm-specific assertions
assertions = [
{
assertion = builtins.elem "amdgpu" config.boot.initrd.kernelModules;
message = "ComfyUI ROCm mode requires 'amdgpu' in boot.initrd.kernelModules";
}
{
assertion = builtins.elem "amdgpu" config.services.xserver.videoDrivers;
message = "ComfyUI ROCm mode requires 'amdgpu' in services.xserver.videoDrivers";
}
{
assertion = config.hardware.amdgpu.opencl.enable;
message = "ComfyUI ROCm mode requires hardware.amdgpu.opencl.enable = true";
}
];
# ROCm and AMD GPU packages
environment.systemPackages = with pkgs; [
rocmPackages.rocm-smi
rocmPackages.rocminfo
rocmPackages.rpp
rocmPackages.rocm-core
rocmPackages.rocm-runtime
rocmPackages.hipblas
rocmPackages.llvm.clang
amdgpu_top
amdctl
];
# Environment variables for ROCm stability
environment.variables = lib.mkMerge [
{
# Stability fix — SDMA is buggy on some AMD unified memory configurations
HSA_ENABLE_SDMA = "0";
}
(lib.mkIf (cfg.rocmOverrideGfx != null) {
HSA_OVERRIDE_GFX_VERSION = cfg.rocmOverrideGfx;
})
];
};
}
@@ -0,0 +1,69 @@
# ComfyUI Docker Configuration for NixOS with GPU support
#
# Usage:
# skyg.nixos.server.services.comfyui = {
# enable = true;
# mode = "rocm"; # or "cuda"
# port = 8188;
# rocmOverrideGfx = "gfx1151"; # optional, for newer AMD GPUs
# };
#
# Run ComfyUI with Docker (ROCm):
# docker run -it --rm \
# --device=/dev/kfd --device=/dev/dri \
# --group-add video --group-add render \
# -v /path/to/models:/app/models \
# -v /path/to/output:/app/output \
# -p 8188:8188 \
# <comfyui-rocm-image>
{ lib, config, ... }:
let
cfg = config.skyg.nixos.server.services.comfyui;
in
{
options.skyg.nixos.server.services.comfyui = with lib; {
enable = mkEnableOption "ComfyUI Docker support with GPU";
mode = mkOption {
type = types.enum [ "rocm" "cuda" ];
default = "rocm";
description = "GPU acceleration mode: 'rocm' for AMD GPUs, 'cuda' for NVIDIA GPUs";
};
port = mkOption {
type = types.port;
default = 8188;
description = "Port for ComfyUI web interface";
};
openFirewall = mkOption {
type = types.bool;
default = true;
description = "Whether to open the firewall for ComfyUI";
};
};
config = lib.mkIf cfg.enable {
# Assertions for required configuration (shared across all modes)
assertions = [
{
assertion = config.virtualisation.docker.enable;
message = "ComfyUI Docker support requires Docker to be enabled (virtualisation.docker.enable = true)";
}
{
assertion = config.hardware.graphics.enable;
message = "ComfyUI Docker support requires graphics to be enabled (hardware.graphics.enable = true)";
}
{
assertion = config.hardware.enableAllFirmware;
message = "ComfyUI Docker support requires firmware to be enabled (hardware.enableAllFirmware = true)";
}
];
# Firewall rules
networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
};
}
@@ -3,6 +3,7 @@
imports = [
./audiobookshelf.nix
./ai-services.nix
./comfyui
./jellyfin.nix
./scrypted.nix
./dockge.nix