diff --git a/hosts/hl-fwdesk.nix b/hosts/hl-fwdesk.nix index 56f3cd0..026672d 100644 --- a/hosts/hl-fwdesk.nix +++ b/hosts/hl-fwdesk.nix @@ -42,13 +42,17 @@ in niri.enable = true; }; }; + server.services.comfyui = { + enable = true; + mode = "rocm"; + port = ports.comfyui; + rocm = { + dataDir = "/data/comfyui"; + }; + }; }; networkDrives = { enable = true; - # options = [ - # "x-systemd.automount" - # "noauto" - # ]; }; }; users.users.ari.extraGroups = [ diff --git a/modules/nixos/server/services/comfyui/cuda.nix b/modules/nixos/server/services/comfyui/cuda.nix new file mode 100644 index 0000000..2170149 --- /dev/null +++ b/modules/nixos/server/services/comfyui/cuda.nix @@ -0,0 +1,23 @@ +{ lib, config, pkgs, ... }: + +let + cfg = config.skyg.nixos.server.services.comfyui; + isCuda = cfg.mode == "cuda"; +in +{ + config = lib.mkIf (cfg.enable && isCuda) { + # TODO: Add CUDA-specific assertions + # assertions = [ + # { + # assertion = config.hardware.nvidia.enable or false; + # message = "ComfyUI CUDA mode requires NVIDIA drivers to be enabled"; + # } + # ]; + + # TODO: Add CUDA-specific packages + # environment.systemPackages = with pkgs; [ + # cudaPackages.cudatoolkit + # nvidia-docker + # ]; + }; +} diff --git a/modules/nixos/server/services/comfyui/default.nix b/modules/nixos/server/services/comfyui/default.nix new file mode 100644 index 0000000..eea1094 --- /dev/null +++ b/modules/nixos/server/services/comfyui/default.nix @@ -0,0 +1,9 @@ +{ ... }: +{ + imports = [ + ./shared.nix + ./rocm.nix + ./rocm-docker.nix + ./cuda.nix + ]; +} diff --git a/modules/nixos/server/services/comfyui/rocm-docker.nix b/modules/nixos/server/services/comfyui/rocm-docker.nix new file mode 100644 index 0000000..ed69aa0 --- /dev/null +++ b/modules/nixos/server/services/comfyui/rocm-docker.nix @@ -0,0 +1,174 @@ +# ComfyUI ROCm Docker Container Build and Service +# +# This module builds the ComfyUI container image and runs it as a systemd service +# using NixOS's OCI container abstraction. + +{ lib, config, pkgs, ... }: + +let + cfg = config.skyg.nixos.server.services.comfyui; + isRocm = cfg.enable && cfg.mode == "rocm"; + + # Load Containerfile from file + containerfile = ./rocm.Containerfile; + + # Image name and tag + imageName = "comfyui-rocm"; + imageTag = cfg.rocm.imageTag; + fullImageName = "${imageName}:${imageTag}"; +in +{ + options.skyg.nixos.server.services.comfyui.rocm = with lib; { + uid = mkOption { + type = types.int; + default = 1000; + description = "User ID to run the ComfyUI container as"; + }; + + gid = mkOption { + type = types.int; + default = 100; + description = "Group ID to run the ComfyUI container as"; + }; + + hsaOverrideGfxVersion = mkOption { + type = types.str; + default = "11.5.1"; + description = "HSA_OVERRIDE_GFX_VERSION for ROCm compatibility"; + }; + + comfyuiCommit = mkOption { + type = types.str; + default = "master"; + description = "ComfyUI git commit or branch to checkout"; + }; + + imageTag = mkOption { + type = types.str; + default = "latest"; + description = "Tag for the built ComfyUI ROCm image"; + }; + + dataDir = mkOption { + type = types.path; + default = "/var/lib/comfyui"; + description = "Directory for ComfyUI data (models, output, etc.)"; + }; + + modelsDir = mkOption { + type = types.nullOr types.path; + default = null; + example = "/mnt/models"; + description = "Optional external models directory to mount"; + }; + + outputDir = mkOption { + type = types.nullOr types.path; + default = null; + example = "/mnt/output"; + description = "Optional external output directory to mount"; + }; + + extraVolumes = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "/path/to/custom:/workspace/ComfyUI/custom_nodes/custom" ]; + description = "Additional volume mounts for the container"; + }; + + autoStart = mkOption { + type = types.bool; + default = true; + description = "Whether to start the ComfyUI container automatically"; + }; + + rebuildImage = mkOption { + type = types.bool; + default = false; + description = "Force rebuild the Docker image on next activation"; + }; + }; + + config = lib.mkIf isRocm { + # Create data directories with proper ownership + systemd.tmpfiles.rules = [ + "d ${cfg.rocm.dataDir} 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/models 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/models/checkpoints 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/models/vae 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/models/loras 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/models/unet 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/models/clip 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/models/controlnet 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/output 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + "d ${cfg.rocm.dataDir}/custom_nodes 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -" + ]; + + # Build script for the Docker image + system.activationScripts.comfyui-rocm-build = { + text = '' + # Check if image exists or rebuild is requested + if ! ${pkgs.docker}/bin/docker image inspect ${fullImageName} > /dev/null 2>&1 || ${lib.boolToString cfg.rocm.rebuildImage}; then + echo "Building ComfyUI ROCm Docker image..." + ${pkgs.docker}/bin/docker build \ + -t ${fullImageName} \ + --build-arg COMFYUI_COMMIT=${cfg.rocm.comfyuiCommit} \ + --build-arg HSA_OVERRIDE_GFX_VERSION=${cfg.rocm.hsaOverrideGfxVersion} \ + -f ${containerfile} \ + ${pkgs.emptyDirectory} + else + echo "ComfyUI ROCm Docker image already exists, skipping build." + fi + ''; + deps = [ ]; + }; + + # OCI container definition + virtualisation.oci-containers.containers.comfyui = { + autoStart = cfg.rocm.autoStart; + image = fullImageName; + + ports = [ + "${toString cfg.port}:8188" + ]; + + # Run as user and ROCm GPU access + user = "${toString cfg.rocm.uid}:${toString cfg.rocm.gid}"; + extraOptions = [ + "--device=/dev/kfd" + "--device=/dev/dri" + "--group-add=video" + "--group-add=render" + "--security-opt=seccomp=unconfined" + "--ipc=host" + ]; + + environment = { + HSA_OVERRIDE_GFX_VERSION = cfg.rocm.hsaOverrideGfxVersion; + HSA_ENABLE_SDMA = "0"; + ROCBLAS_USE_HIPBLASLT = "1"; + PYTORCH_HIP_ALLOC_CONF = "expandable_segments:False"; + }; + + volumes = [ + "${cfg.rocm.dataDir}/custom_nodes:/workspace/ComfyUI/custom_nodes" + ] ++ (if cfg.rocm.modelsDir != null + then [ "${cfg.rocm.modelsDir}:/workspace/ComfyUI/models" ] + else [ "${cfg.rocm.dataDir}/models:/workspace/ComfyUI/models" ]) + ++ (if cfg.rocm.outputDir != null + then [ "${cfg.rocm.outputDir}:/workspace/ComfyUI/output" ] + else [ "${cfg.rocm.dataDir}/output:/workspace/ComfyUI/output" ]) + ++ cfg.rocm.extraVolumes; + }; + + # Ensure the container service starts after Docker image is built + systemd.services."docker-comfyui" = { + after = [ "docker.service" ]; + requires = [ "docker.service" ]; + serviceConfig = { + Restart = "on-failure"; + RestartSec = "10s"; + }; + }; + }; +} diff --git a/modules/nixos/server/services/comfyui/rocm.Containerfile b/modules/nixos/server/services/comfyui/rocm.Containerfile new file mode 100644 index 0000000..8bd1774 --- /dev/null +++ b/modules/nixos/server/services/comfyui/rocm.Containerfile @@ -0,0 +1,58 @@ +FROM rocm/pytorch:rocm7.2.1_ubuntu24.04_py3.12_pytorch_release_2.9.1 + +# ─── ROCm environment for gfx1151 (Strix Halo) ─────────────────────────── +ENV HSA_OVERRIDE_GFX_VERSION=11.5.1 +ENV HSA_ENABLE_SDMA=0 +ENV ROCBLAS_USE_HIPBLASLT=1 +ENV PYTORCH_HIP_ALLOC_CONF=expandable_segments:False + +# ─── Clone ComfyUI ──────────────────────────────────────────────────────── +ARG COMFYUI_COMMIT=master +ENV COMFYUI_PATH=/workspace/ComfyUI + +RUN git clone https://github.com/comfyanonymous/ComfyUI "$COMFYUI_PATH" \ + && cd "$COMFYUI_PATH" \ + && if [ "$COMFYUI_COMMIT" != "master" ]; then git checkout "$COMFYUI_COMMIT"; fi + +# ─── Install ComfyUI requirements ───────────────────────────────────────── +RUN pip install --no-cache-dir -r "$COMFYUI_PATH/requirements.txt" + +# ─── Custom nodes ───────────────────────────────────────────────────────── + +# ComfyUI-Manager +RUN git clone https://github.com/ltdrdata/ComfyUI-Manager \ + "$COMFYUI_PATH/custom_nodes/ComfyUI-Manager" \ + && pip install --no-cache-dir \ + -r "$COMFYUI_PATH/custom_nodes/ComfyUI-Manager/requirements.txt" + +# WAN 2.1 video nodes +RUN git clone https://github.com/kijai/ComfyUI-WanVideoWrapper \ + "$COMFYUI_PATH/custom_nodes/ComfyUI-WanVideoWrapper" \ + && pip install --no-cache-dir \ + -r "$COMFYUI_PATH/custom_nodes/ComfyUI-WanVideoWrapper/requirements.txt" \ + || true + +# Video Helper Suite +RUN git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite \ + "$COMFYUI_PATH/custom_nodes/ComfyUI-VideoHelperSuite" \ + && pip install --no-cache-dir \ + -r "$COMFYUI_PATH/custom_nodes/ComfyUI-VideoHelperSuite/requirements.txt" \ + || true + +# ─── Model directories ──────────────────────────────────────────────────── +RUN mkdir -p \ + "$COMFYUI_PATH/models/checkpoints" \ + "$COMFYUI_PATH/models/vae" \ + "$COMFYUI_PATH/models/loras" \ + "$COMFYUI_PATH/models/unet" \ + "$COMFYUI_PATH/models/clip" \ + "$COMFYUI_PATH/models/controlnet" \ + "$COMFYUI_PATH/output" + +WORKDIR "$COMFYUI_PATH" +EXPOSE 8188 + +CMD ["python", "main.py", \ + "--listen", "0.0.0.0", \ + "--port", "8188", \ + "--use-pytorch-cross-attention"] diff --git a/modules/nixos/server/services/comfyui/rocm.nix b/modules/nixos/server/services/comfyui/rocm.nix new file mode 100644 index 0000000..5b83d6f --- /dev/null +++ b/modules/nixos/server/services/comfyui/rocm.nix @@ -0,0 +1,58 @@ +{ lib, config, pkgs, ... }: + +let + cfg = config.skyg.nixos.server.services.comfyui; + isRocm = cfg.enable && cfg.mode == "rocm"; +in +{ + options.skyg.nixos.server.services.comfyui = with lib; { + rocmOverrideGfx = mkOption { + type = types.nullOr types.str; + default = null; + example = "gfx1151"; + description = "Override GFX version for ROCm (useful for newer AMD GPUs)"; + }; + }; + + config = lib.mkIf isRocm { + # ROCm-specific assertions + assertions = [ + { + assertion = builtins.elem "amdgpu" config.boot.initrd.kernelModules; + message = "ComfyUI ROCm mode requires 'amdgpu' in boot.initrd.kernelModules"; + } + { + assertion = builtins.elem "amdgpu" config.services.xserver.videoDrivers; + message = "ComfyUI ROCm mode requires 'amdgpu' in services.xserver.videoDrivers"; + } + { + assertion = config.hardware.amdgpu.opencl.enable; + message = "ComfyUI ROCm mode requires hardware.amdgpu.opencl.enable = true"; + } + ]; + + # ROCm and AMD GPU packages + environment.systemPackages = with pkgs; [ + rocmPackages.rocm-smi + rocmPackages.rocminfo + rocmPackages.rpp + rocmPackages.rocm-core + rocmPackages.rocm-runtime + rocmPackages.hipblas + rocmPackages.llvm.clang + amdgpu_top + amdctl + ]; + + # Environment variables for ROCm stability + environment.variables = lib.mkMerge [ + { + # Stability fix — SDMA is buggy on some AMD unified memory configurations + HSA_ENABLE_SDMA = "0"; + } + (lib.mkIf (cfg.rocmOverrideGfx != null) { + HSA_OVERRIDE_GFX_VERSION = cfg.rocmOverrideGfx; + }) + ]; + }; +} diff --git a/modules/nixos/server/services/comfyui/shared.nix b/modules/nixos/server/services/comfyui/shared.nix new file mode 100644 index 0000000..0435457 --- /dev/null +++ b/modules/nixos/server/services/comfyui/shared.nix @@ -0,0 +1,69 @@ +# ComfyUI Docker Configuration for NixOS with GPU support +# +# Usage: +# skyg.nixos.server.services.comfyui = { +# enable = true; +# mode = "rocm"; # or "cuda" +# port = 8188; +# rocmOverrideGfx = "gfx1151"; # optional, for newer AMD GPUs +# }; +# +# Run ComfyUI with Docker (ROCm): +# docker run -it --rm \ +# --device=/dev/kfd --device=/dev/dri \ +# --group-add video --group-add render \ +# -v /path/to/models:/app/models \ +# -v /path/to/output:/app/output \ +# -p 8188:8188 \ +# + +{ lib, config, ... }: + +let + cfg = config.skyg.nixos.server.services.comfyui; +in +{ + options.skyg.nixos.server.services.comfyui = with lib; { + enable = mkEnableOption "ComfyUI Docker support with GPU"; + + mode = mkOption { + type = types.enum [ "rocm" "cuda" ]; + default = "rocm"; + description = "GPU acceleration mode: 'rocm' for AMD GPUs, 'cuda' for NVIDIA GPUs"; + }; + + port = mkOption { + type = types.port; + default = 8188; + description = "Port for ComfyUI web interface"; + }; + + openFirewall = mkOption { + type = types.bool; + default = true; + description = "Whether to open the firewall for ComfyUI"; + }; + }; + + config = lib.mkIf cfg.enable { + # Assertions for required configuration (shared across all modes) + assertions = [ + { + assertion = config.virtualisation.docker.enable; + message = "ComfyUI Docker support requires Docker to be enabled (virtualisation.docker.enable = true)"; + } + { + assertion = config.hardware.graphics.enable; + message = "ComfyUI Docker support requires graphics to be enabled (hardware.graphics.enable = true)"; + } + { + assertion = config.hardware.enableAllFirmware; + message = "ComfyUI Docker support requires firmware to be enabled (hardware.enableAllFirmware = true)"; + } + ]; + + # Firewall rules + networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ]; + networking.firewall.allowedUDPPorts = lib.mkIf cfg.openFirewall [ cfg.port ]; + }; +} diff --git a/modules/nixos/server/services/default.nix b/modules/nixos/server/services/default.nix index e3b83db..438e876 100644 --- a/modules/nixos/server/services/default.nix +++ b/modules/nixos/server/services/default.nix @@ -3,6 +3,7 @@ imports = [ ./audiobookshelf.nix ./ai-services.nix + ./comfyui ./jellyfin.nix ./scrypted.nix ./dockge.nix