Files
nixos-setup/modules/nixos/server/services/comfyui/rocm-docker.nix
T
2026-04-15 13:34:20 -04:00

177 lines
6.1 KiB
Nix

# ComfyUI ROCm Docker Container Build and Service
#
# This module builds the ComfyUI container image and runs it as a systemd service
# using NixOS's OCI container abstraction.
{ lib, config, pkgs, ... }:
let
cfg = config.skyg.nixos.server.services.comfyui;
isRocm = cfg.enable && cfg.mode == "rocm";
# Load Containerfile from file
containerfile = ./rocm.Containerfile;
# Image name and tag
imageName = "comfyui-rocm";
imageTag = cfg.rocm.imageTag;
fullImageName = "${imageName}:${imageTag}";
in
{
options.skyg.nixos.server.services.comfyui.rocm = with lib; {
uid = mkOption {
type = types.int;
default = 1000;
description = "User ID to run the ComfyUI container as";
};
gid = mkOption {
type = types.int;
default = 100;
description = "Group ID to run the ComfyUI container as";
};
hsaOverrideGfxVersion = mkOption {
type = types.str;
default = "11.5.1";
description = "HSA_OVERRIDE_GFX_VERSION for ROCm compatibility";
};
comfyuiCommit = mkOption {
type = types.str;
default = "master";
description = "ComfyUI git commit or branch to checkout";
};
imageTag = mkOption {
type = types.str;
default = "latest";
description = "Tag for the built ComfyUI ROCm image";
};
dataDir = mkOption {
type = types.path;
default = "/var/lib/comfyui";
description = "Directory for ComfyUI data (models, output, etc.)";
};
modelsDir = mkOption {
type = types.nullOr types.path;
default = null;
example = "/mnt/models";
description = "Optional external models directory to mount";
};
outputDir = mkOption {
type = types.nullOr types.path;
default = null;
example = "/mnt/output";
description = "Optional external output directory to mount";
};
extraVolumes = mkOption {
type = types.listOf types.str;
default = [ ];
example = [ "/path/to/custom:/workspace/ComfyUI/custom_nodes/custom" ];
description = "Additional volume mounts for the container";
};
autoStart = mkOption {
type = types.bool;
default = true;
description = "Whether to start the ComfyUI container automatically";
};
rebuildImage = mkOption {
type = types.bool;
default = false;
description = "Force rebuild the Docker image on next activation";
};
};
config = lib.mkIf isRocm {
# Create data directories with proper ownership
systemd.tmpfiles.rules = [
"d ${cfg.rocm.dataDir} 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/checkpoints 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/vae 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/loras 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/unet 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/clip 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/models/controlnet 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/output 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/custom_nodes 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/user 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/input 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
"d ${cfg.rocm.dataDir}/temp 0755 ${toString cfg.rocm.uid} ${toString cfg.rocm.gid} -"
];
# Build script for the Docker image
system.activationScripts.comfyui-rocm-build = {
text = ''
# Check if image exists or rebuild is requested
if ! ${pkgs.docker}/bin/docker image inspect ${fullImageName} > /dev/null 2>&1 || ${lib.boolToString cfg.rocm.rebuildImage}; then
echo "Building ComfyUI ROCm Docker image..."
${pkgs.docker}/bin/docker build \
-t ${fullImageName} \
--build-arg COMFYUI_COMMIT=${cfg.rocm.comfyuiCommit} \
--build-arg HSA_OVERRIDE_GFX_VERSION=${cfg.rocm.hsaOverrideGfxVersion} \
-f ${containerfile} \
${pkgs.emptyDirectory}
else
echo "ComfyUI ROCm Docker image already exists, skipping build."
fi
'';
deps = [ ];
};
# OCI container definition
virtualisation.oci-containers.containers.comfyui = {
autoStart = cfg.rocm.autoStart;
image = fullImageName;
ports = [
"${toString cfg.port}:8188"
];
# Run as user and ROCm GPU access
user = "${toString cfg.rocm.uid}:${toString cfg.rocm.gid}";
extraOptions = [
"--device=/dev/kfd"
"--device=/dev/dri"
"--group-add=video"
"--group-add=render"
"--security-opt=seccomp=unconfined"
"--ipc=host"
];
environment = {
HSA_OVERRIDE_GFX_VERSION = cfg.rocm.hsaOverrideGfxVersion;
HSA_ENABLE_SDMA = "0";
ROCBLAS_USE_HIPBLASLT = "1";
PYTORCH_HIP_ALLOC_CONF = "expandable_segments:False";
};
volumes = [
"${cfg.rocm.dataDir}/custom_nodes:/workspace/ComfyUI/custom_nodes"
"${cfg.rocm.dataDir}/user:/workspace/ComfyUI/user"
"${cfg.rocm.dataDir}/input:/workspace/ComfyUI/input"
"${cfg.rocm.dataDir}/temp:/workspace/ComfyUI/temp"
] ++ (if cfg.rocm.modelsDir != null
then [ "${cfg.rocm.modelsDir}:/workspace/ComfyUI/models" ]
else [ "${cfg.rocm.dataDir}/models:/workspace/ComfyUI/models" ])
++ (if cfg.rocm.outputDir != null
then [ "${cfg.rocm.outputDir}:/workspace/ComfyUI/output" ]
else [ "${cfg.rocm.dataDir}/output:/workspace/ComfyUI/output" ])
++ cfg.rocm.extraVolumes;
};
# Ensure the container service starts after Docker image is built
systemd.services."docker-comfyui" = {
after = [ "docker.service" ];
requires = [ "docker.service" ];
};
};
}