From a27d701bac9a78cb495e07b3a4124eb9e10b8dfc Mon Sep 17 00:00:00 2001 From: Ari Sosnovsky Date: Fri, 8 Nov 2024 22:22:09 -0500 Subject: [PATCH] k8s cluster working & update --- flake.lock | 24 ++++++++++++------------ hosts/hl-minipc1.nix | 3 ++- hosts/hl-minipc2.nix | 2 +- hosts/hl-terra1.nix | 8 +------- hosts/scripts/fwbook/functions.sh | 6 +++++- modules/home/default.nix | 1 + modules/main.nix | 4 ++-- modules/nixos/server/k8s/default.nix | 3 ++- modules/nixos/server/k8s/master.nix | 8 +++++++- modules/nixos/server/k8s/node.nix | 2 +- 10 files changed, 34 insertions(+), 27 deletions(-) diff --git a/flake.lock b/flake.lock index 7a827f4..0472716 100644 --- a/flake.lock +++ b/flake.lock @@ -304,11 +304,11 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1730079405, - "narHash": "sha256-kvq8Iq1P7Ut/cu0VWn8h+gnHTPWVdPhnaAiIKHY4vJs=", + "lastModified": 1731116135, + "narHash": "sha256-IJ7SViFR2APiuV3k+8v6NIMI1K7oP0Q3cNNfrdlbefU=", "owner": "lilyinstarlight", "repo": "nixos-cosmic", - "rev": "965b4d7b838dd4f37293c3b8160c64c2edcfb226", + "rev": "c74c207183e32f23a94b4bb8b69bc5064db98894", "type": "github" }, "original": { @@ -367,11 +367,11 @@ }, "nixpkgs-stable_2": { "locked": { - "lastModified": 1729973466, - "narHash": "sha256-knnVBGfTCZlQgxY1SgH0vn2OyehH9ykfF8geZgS95bk=", + "lastModified": 1730883749, + "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cd3e8833d70618c4eea8df06f95b364b016d4950", + "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede", "type": "github" }, "original": { @@ -426,11 +426,11 @@ ] }, "locked": { - "lastModified": 1729996302, - "narHash": "sha256-QEU1NQq1+7s1na69Chig9K0iDDTKN0O4Zreo9A9rccA=", + "lastModified": 1731032894, + "narHash": "sha256-dQSyYPmrQiPr+PGEd+K8038rubFGz7G/dNXVeaGWE0w=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "a1b337569f334ff0a01b57627f17b201d746d24c", + "rev": "d52f2a4c103a0acf09ded857b9e2519ae2360e59", "type": "github" }, "original": { @@ -456,11 +456,11 @@ }, "unstable": { "locked": { - "lastModified": 1730048550, - "narHash": "sha256-VMt8RaCz0nkMcpAjthTjZZE14hYbQy3Y5FFMKus0XYo=", + "lastModified": 1731115093, + "narHash": "sha256-vDITOvO17TymGq25e3Seegq74CsqUh6EjVvAIGGbDqM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0b2e723efac0720376031252fc500065121945d9", + "rev": "df76cd6ae1b33d47b3b7040915737e5c71ea76bb", "type": "github" }, "original": { diff --git a/hosts/hl-minipc1.nix b/hosts/hl-minipc1.nix index f6c75b7..5553c70 100644 --- a/hosts/hl-minipc1.nix +++ b/hosts/hl-minipc1.nix @@ -56,7 +56,8 @@ in services.kubernetes.kubelet.hostname = "minipc1.lab.internal"; skyg.nixos.server.k8s = { enable = true; - isMaster = false; + isMaster = true; + isNode = true; }; } diff --git a/hosts/hl-minipc2.nix b/hosts/hl-minipc2.nix index 6becfb3..52c84f6 100644 --- a/hosts/hl-minipc2.nix +++ b/hosts/hl-minipc2.nix @@ -36,7 +36,7 @@ services.kubernetes.kubelet.hostname = "minipc2.lab.internal"; skyg.nixos.server.k8s = { enable = true; - isMaster = false; + isNode = true; }; } diff --git a/hosts/hl-terra1.nix b/hosts/hl-terra1.nix index 3faa0c8..91d2440 100644 --- a/hosts/hl-terra1.nix +++ b/hosts/hl-terra1.nix @@ -1,5 +1,5 @@ { user }: -{ pkgs, lib, config, ... }: +{ pkgs, ... }: let openPorts = [ # nfs @@ -40,11 +40,5 @@ in /mnt/Data/apps 10.0.0.0/16(rw,wdelay,insecure,no_root_squash,no_subtree_check,sec=sys,rw,insecure,no_root_squash,no_all_squash) ''; }; - # K8s - services.kubernetes.kubelet.hostname = "terra1.lab.internal"; - skyg.nixos.server.k8s = { - enable = true; - isMaster = true; - }; } diff --git a/hosts/scripts/fwbook/functions.sh b/hosts/scripts/fwbook/functions.sh index 112aa2b..88faaa8 100755 --- a/hosts/scripts/fwbook/functions.sh +++ b/hosts/scripts/fwbook/functions.sh @@ -18,4 +18,8 @@ function start-windows-app() { # function clean-file() { # ${@} | grep -v '^\s*#' | grep -v '^$' -# } \ No newline at end of file +# } + +function clean-k8s-install() { + sudo rm -rf /var/lib/kubernetes/ /var/lib/etcd/ /var/lib/cfssl/ /var/lib/kubelet/ /etc/kube-flannel/ /etc/kubernetes/ +} \ No newline at end of file diff --git a/modules/home/default.nix b/modules/home/default.nix index 5cbb20c..4b2ed16 100644 --- a/modules/home/default.nix +++ b/modules/home/default.nix @@ -52,6 +52,7 @@ in htop nnn thefuck + fastfetch ] ++ (if enableDevelopmentKit then [ devenv diff --git a/modules/main.nix b/modules/main.nix index 5c68ee0..6889f43 100644 --- a/modules/main.nix +++ b/modules/main.nix @@ -25,8 +25,8 @@ skyg.networkDrives.enable = enableNetworkDrives; skyg.nixos.common.containers = (if os.enable then os.containers else { }); skyg.nixos.server.k8s = { - masterIP = "10.0.10.8"; - masterHostName = "terra1.lab.internal"; + masterIP = "10.0.10.6"; + masterHostName = "minipc1.lab.internal"; }; system.stateVersion = systemStateVersion; nixpkgs.config.allowUnfree = true; diff --git a/modules/nixos/server/k8s/default.nix b/modules/nixos/server/k8s/default.nix index fa1ea5b..dec9b67 100644 --- a/modules/nixos/server/k8s/default.nix +++ b/modules/nixos/server/k8s/default.nix @@ -13,6 +13,8 @@ in "Enable K8s"; isMaster = lib.mkEnableOption "Enable as master"; + isNode = lib.mkEnableOption + "Enable as node"; masterIP = lib.mkOption { type = lib.types.str; }; @@ -33,7 +35,6 @@ in 10250 # kubelet api 10259 # kubelet scheduler 10257 # kubelet controller - 8888 # no idea what this is ]; portRanges = [ { from = 2379; to = 2380; } # etcd diff --git a/modules/nixos/server/k8s/master.nix b/modules/nixos/server/k8s/master.nix index f9e89bc..c085699 100644 --- a/modules/nixos/server/k8s/master.nix +++ b/modules/nixos/server/k8s/master.nix @@ -4,8 +4,14 @@ let in { config = lib.mkIf (cfg.enable && cfg.isMaster) { + networking.firewall.allowedUDPPorts = [ + 8888 # open cfssl + ]; + networking.firewall.allowedTCPPorts = [ + 8888 # open cfssl + ]; services.kubernetes = { - roles = [ "master" ]; + roles = if cfg.isNode then [ "master" "node" ] else [ "master" ]; apiserver = { securePort = cfg.masterAPIPort; advertiseAddress = cfg.masterIP; diff --git a/modules/nixos/server/k8s/node.nix b/modules/nixos/server/k8s/node.nix index 5500429..eb66e01 100644 --- a/modules/nixos/server/k8s/node.nix +++ b/modules/nixos/server/k8s/node.nix @@ -3,7 +3,7 @@ let cfg = config.skyg.nixos.server.k8s; in { - config = lib.mkIf (cfg.enable && (!cfg.isMaster)) { + config = lib.mkIf (cfg.enable && (!cfg.isMaster) && cfg.isNode) { services.kubernetes = { roles = [ "node" ]; kubelet.kubeconfig.server = "https://${cfg.masterHostName}:${toString cfg.masterAPIPort}";