From 3f1b8a2b3e9e2ab70af1c1a0cd8f453437e7a4c9 Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Wed, 30 Oct 2024 17:10:38 -0400 Subject: [PATCH] Making sure that k8s is properly distributed --- hosts/hl-minipc1.nix | 3 +- hosts/hl-minipc2.nix | 1 + hosts/hl-terra1.nix | 6 ++++ modules/main.nix | 4 +-- modules/nixos/server/k8s/default.nix | 49 +++++++++++++++++++--------- modules/nixos/server/k8s/master.nix | 4 +-- 6 files changed, 45 insertions(+), 22 deletions(-) diff --git a/hosts/hl-minipc1.nix b/hosts/hl-minipc1.nix index 1589915..f6c75b7 100644 --- a/hosts/hl-minipc1.nix +++ b/hosts/hl-minipc1.nix @@ -53,9 +53,10 @@ in networking.firewall.allowedUDPPorts = openPorts; networking.firewall.allowedTCPPorts = openPorts; # K8s + services.kubernetes.kubelet.hostname = "minipc1.lab.internal"; skyg.nixos.server.k8s = { enable = true; - isMaster = true; + isMaster = false; }; } diff --git a/hosts/hl-minipc2.nix b/hosts/hl-minipc2.nix index 659f613..6becfb3 100644 --- a/hosts/hl-minipc2.nix +++ b/hosts/hl-minipc2.nix @@ -33,6 +33,7 @@ }; }; # K8s + services.kubernetes.kubelet.hostname = "minipc2.lab.internal"; skyg.nixos.server.k8s = { enable = true; isMaster = false; diff --git a/hosts/hl-terra1.nix b/hosts/hl-terra1.nix index 2e452e0..3faa0c8 100644 --- a/hosts/hl-terra1.nix +++ b/hosts/hl-terra1.nix @@ -40,5 +40,11 @@ in /mnt/Data/apps 10.0.0.0/16(rw,wdelay,insecure,no_root_squash,no_subtree_check,sec=sys,rw,insecure,no_root_squash,no_all_squash) ''; }; + # K8s + services.kubernetes.kubelet.hostname = "terra1.lab.internal"; + skyg.nixos.server.k8s = { + enable = true; + isMaster = true; + }; } diff --git a/modules/main.nix b/modules/main.nix index 6889f43..5c68ee0 100644 --- a/modules/main.nix +++ b/modules/main.nix @@ -25,8 +25,8 @@ skyg.networkDrives.enable = enableNetworkDrives; skyg.nixos.common.containers = (if os.enable then os.containers else { }); skyg.nixos.server.k8s = { - masterIP = "10.0.10.6"; - masterHostName = "minipc1.lab.internal"; + masterIP = "10.0.10.8"; + masterHostName = "terra1.lab.internal"; }; system.stateVersion = systemStateVersion; nixpkgs.config.allowUnfree = true; diff --git a/modules/nixos/server/k8s/default.nix b/modules/nixos/server/k8s/default.nix index 60cc97d..fa1ea5b 100644 --- a/modules/nixos/server/k8s/default.nix +++ b/modules/nixos/server/k8s/default.nix @@ -26,22 +26,39 @@ in }; }; - config = lib.mkIf cfg.enable { - # resolve master hostname - networking.extraHosts = "${cfg.masterIP} ${cfg.masterHostName}"; - - # packages for administration tasks - environment.systemPackages = with pkgs; [ - kubectl - kubernetes - ]; - services.kubernetes = { - masterAddress = cfg.masterHostName; - easyCerts = true; - apiserverAddress = "https://${cfg.masterHostName}:${toString cfg.masterAPIPort}"; - addons.dns.enable = true; - kubelet.extraOpts = "--fail-swap-on=false"; + config = + let + ports = [ + cfg.masterAPIPort # k8s api server + 10250 # kubelet api + 10259 # kubelet scheduler + 10257 # kubelet controller + 8888 # no idea what this is + ]; + portRanges = [ + { from = 2379; to = 2380; } # etcd + ]; + in + lib.mkIf cfg.enable { + # resolve master hostname + networking.extraHosts = "${cfg.masterIP} ${cfg.masterHostName}"; + networking.firewall.allowedUDPPorts = ports; + networking.firewall.allowedUDPPortRanges = portRanges; + networking.firewall.allowedTCPPorts = ports; + networking.firewall.allowedTCPPortRanges = portRanges; + # packages for administration tasks + environment.systemPackages = with pkgs; [ + kubectl + kubernetes + ]; + services.kubernetes = { + proxy.hostname = cfg.masterHostName; + masterAddress = cfg.masterHostName; + easyCerts = true; + apiserverAddress = "https://${cfg.masterHostName}:${toString cfg.masterAPIPort}"; + addons.dns.enable = true; + kubelet.extraOpts = "--fail-swap-on=false"; + }; }; - }; } diff --git a/modules/nixos/server/k8s/master.nix b/modules/nixos/server/k8s/master.nix index 31a5e95..f9e89bc 100644 --- a/modules/nixos/server/k8s/master.nix +++ b/modules/nixos/server/k8s/master.nix @@ -5,13 +5,11 @@ in { config = lib.mkIf (cfg.enable && cfg.isMaster) { services.kubernetes = { - roles = [ "master" "node" ]; + roles = [ "master" ]; apiserver = { securePort = cfg.masterAPIPort; advertiseAddress = cfg.masterIP; }; }; - networking.firewall.allowedUDPPorts = [ cfg.masterAPIPort ]; - networking.firewall.allowedTCPPorts = [ cfg.masterAPIPort ]; }; }