From c8aaf743b94edad7ab2b9f870034fcc1ec49e408 Mon Sep 17 00:00:00 2001 From: Ariel Sosnovsky Date: Mon, 20 May 2024 15:43:48 -0400 Subject: [PATCH] Added sumo-mac --- configs/sumo/functions.sh | 55 +++++++++++++++++++ configs/sumo/scripts/airflow | 4 ++ configs/sumo/scripts/airflow-v2 | 4 ++ configs/sumo/scripts/airflow-v2-list-tasks | 4 ++ .../airflow-v2-restart-web-and-scheduler | 7 +++ configs/sumo/scripts/analytics | 4 ++ configs/sumo/scripts/csi | 4 ++ configs/sumo/scripts/docker-login-sumo-ecr | 10 ++++ configs/sumo/scripts/eks | 6 ++ configs/sumo/scripts/eks-namespace | 6 ++ configs/sumo/scripts/get-memory-dump | 25 +++++++++ configs/sumo/scripts/helm-out | 25 +++++++++ configs/sumo/scripts/identify-perm | 9 +++ configs/sumo/scripts/mount-dev | 6 ++ configs/sumo/scripts/open-aws | 5 ++ configs/sumo/scripts/regex | 3 + configs/sumo/scripts/sagemaker | 3 + configs/sumo/scripts/update-aws-cred | 32 +++++++++++ configs/sumo/scripts/update-dsh-knobs | 8 +++ flake.nix | 20 ++++++- hosts/asosnovsky-mac.nix | 47 ++++++++++++++++ 21 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 configs/sumo/functions.sh create mode 100755 configs/sumo/scripts/airflow create mode 100755 configs/sumo/scripts/airflow-v2 create mode 100755 configs/sumo/scripts/airflow-v2-list-tasks create mode 100755 configs/sumo/scripts/airflow-v2-restart-web-and-scheduler create mode 100755 configs/sumo/scripts/analytics create mode 100755 configs/sumo/scripts/csi create mode 100755 configs/sumo/scripts/docker-login-sumo-ecr create mode 100755 configs/sumo/scripts/eks create mode 100755 configs/sumo/scripts/eks-namespace create mode 100755 configs/sumo/scripts/get-memory-dump create mode 100755 configs/sumo/scripts/helm-out create mode 100755 configs/sumo/scripts/identify-perm create mode 100755 configs/sumo/scripts/mount-dev create mode 100755 configs/sumo/scripts/open-aws create mode 100755 configs/sumo/scripts/regex create mode 100755 configs/sumo/scripts/sagemaker create mode 100755 configs/sumo/scripts/update-aws-cred create mode 100755 configs/sumo/scripts/update-dsh-knobs create mode 100644 hosts/asosnovsky-mac.nix diff --git a/configs/sumo/functions.sh b/configs/sumo/functions.sh new file mode 100644 index 0000000..6536c68 --- /dev/null +++ b/configs/sumo/functions.sh @@ -0,0 +1,55 @@ +export DEV_DEPLOYMENTS=(stag long) +export PROD_DEPLOYMENTS=(dub fra mon mum prod syd tky us2 fed) +echo "Loading functions" +function airflow-v2-exec-web() { + dep=${1} + airflow-v2-exec $dep bash +} +function airflow-v2-exec() { + dep=${1} + airflow-v2 $dep exec -it svc/$dep-analytics-airflow-v2-webserver -c webserver -- ${@:2} +} +function tsh.login() { + tsh login -d --proxy teleport.kumoroku.com eks-${1} + tsh kube login eks-${1} +} +function tsh.hop() { + tsh ssh -d --cluster=eks-${1} hop-${2:-1} +} +function tsh.shared() { + tsh ssh -d --cluster=main shared_box-${2:-1} +} +function tsh.login.all() { + for dep in ${DEV_DEPLOYMENTS[@]}; do + echo "---------" + echo " $dep " + echo "---------" + tsh.login $dep + done + for dep in ${PROD_DEPLOYMENTS[@]}; do + echo "---------" + echo " $dep " + echo "---------" + tsh.login $dep + done +} +function airflow-v2-all-tasks() { + for dep in ${DEV_DEPLOYMENTS[@]}; do + echo "[ $dep ]" + airflow-v2-list-tasks $dep + done + for dep in ${PROD_DEPLOYMENTS[@]}; do + echo "[ $dep ]" + airflow-v2-list-tasks $dep + done +} +function airflow-v2-delete-pod-tasks() { + dep=${1} + for pod in $(airflow-v2-list-tasks $dep | awk -F ' ' '{print $1}'); do airflow-v2 $dep delete pod/$pod &; done +} +function open-paas() { + dep=${1:-infrastructure} + eval $(dev hcvault cli_env -a $dep) + export GITHUB_TOKEN=$(dev secrets get -s github_token | jq -r .github_token_password) + paas-iac-shell +} diff --git a/configs/sumo/scripts/airflow b/configs/sumo/scripts/airflow new file mode 100755 index 0000000..e395ab5 --- /dev/null +++ b/configs/sumo/scripts/airflow @@ -0,0 +1,4 @@ +#!/bin/bash + +dep=${1} +eks $dep analytics-airflow ${@:2} \ No newline at end of file diff --git a/configs/sumo/scripts/airflow-v2 b/configs/sumo/scripts/airflow-v2 new file mode 100755 index 0000000..ff27dcd --- /dev/null +++ b/configs/sumo/scripts/airflow-v2 @@ -0,0 +1,4 @@ +#!/bin/bash + +dep=${1} +eks $dep analytics-airflow-v2 ${@:2} diff --git a/configs/sumo/scripts/airflow-v2-list-tasks b/configs/sumo/scripts/airflow-v2-list-tasks new file mode 100755 index 0000000..0398695 --- /dev/null +++ b/configs/sumo/scripts/airflow-v2-list-tasks @@ -0,0 +1,4 @@ +#!/bin/bash + +current_dep=$(kubectl config current-context | cut -d '-' -f2) +airflow-v2 ${1:-$current_dep} get pod --sort-by=.status.startTime -l task_id \ No newline at end of file diff --git a/configs/sumo/scripts/airflow-v2-restart-web-and-scheduler b/configs/sumo/scripts/airflow-v2-restart-web-and-scheduler new file mode 100755 index 0000000..6f4ea1b --- /dev/null +++ b/configs/sumo/scripts/airflow-v2-restart-web-and-scheduler @@ -0,0 +1,7 @@ +#!/bin/bash + +current_dep=$(kubectl config current-context | cut -d '-' -f2) +dep=${1:-$current_dep} +airflow-v2 $dep rollout restart deploy/$dep-analytics-airflow-v2-web & +airflow-v2 $dep rollout restart deploy/$dep-analytics-airflow-v2-scheduler & +wait diff --git a/configs/sumo/scripts/analytics b/configs/sumo/scripts/analytics new file mode 100755 index 0000000..ca1b191 --- /dev/null +++ b/configs/sumo/scripts/analytics @@ -0,0 +1,4 @@ +#!/bin/bash + +dep=${1} +eks $dep analytics ${@:2} \ No newline at end of file diff --git a/configs/sumo/scripts/csi b/configs/sumo/scripts/csi new file mode 100755 index 0000000..d059a0f --- /dev/null +++ b/configs/sumo/scripts/csi @@ -0,0 +1,4 @@ +#!/bin/bash + +dep=${1} +eks $dep csi ${@:2} \ No newline at end of file diff --git a/configs/sumo/scripts/docker-login-sumo-ecr b/configs/sumo/scripts/docker-login-sumo-ecr new file mode 100755 index 0000000..f436038 --- /dev/null +++ b/configs/sumo/scripts/docker-login-sumo-ecr @@ -0,0 +1,10 @@ +#!/bin/bash + + +dev hcvault login +eval $(dev hcvault cli_env -a infrastructure) + +region="us-west-2" +aws_account_id=`aws sts get-caller-identity --query "Account" --output text` + +aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${aws_account_id}.dkr.ecr.${region}.amazonaws.com diff --git a/configs/sumo/scripts/eks b/configs/sumo/scripts/eks new file mode 100755 index 0000000..94c6621 --- /dev/null +++ b/configs/sumo/scripts/eks @@ -0,0 +1,6 @@ +#!/bin/bash + +dep=${1} +assembly=${2} +echo "${dep}/${assembly}" +kubectl --cluster eks-$dep -n $dep-${assembly} --context eks-$dep-eks-$dep ${@:3} diff --git a/configs/sumo/scripts/eks-namespace b/configs/sumo/scripts/eks-namespace new file mode 100755 index 0000000..ef41b66 --- /dev/null +++ b/configs/sumo/scripts/eks-namespace @@ -0,0 +1,6 @@ +#!/bin/bash + +current_dep=$(kubectl config current-context | cut -d "-" -f2) +dep=${2:-$current_dep} +echo "$dep $current_dep" +kubectl config set-context --current --namespace=$dep-${1} \ No newline at end of file diff --git a/configs/sumo/scripts/get-memory-dump b/configs/sumo/scripts/get-memory-dump new file mode 100755 index 0000000..ccb6d1c --- /dev/null +++ b/configs/sumo/scripts/get-memory-dump @@ -0,0 +1,25 @@ + +# pod=analytics-84d4bff6c-2fzj9 +namespace=us2-analytics + +echo "getting pods..." +pods=`kubectl get pods -n $namespace` +totalpods=$(echo $pods | wc -l) +totalpods="$((totalpods-1))" +echo $pods | awk "NR==1" | awk '$0=" "$0' +echo $pods | awk "NR>1" | awk '$0=NR") "$0' +echo "> Please select one of the pods, enter a number between 1 and $totalpods" +read pod_num + +case ${pod_num#[-+]} in + *[!0-9]* | '') echo "Must be an integer." ;; +esac + +pod=$(kubectl get pods -n $namespace | awk {'print $1}' | awk -v pod_num=$pod_num 'NR==pod_num+1') +echo "Getting memory dump for $pod - Continue? [ANY KEY]..." +read +kubectl exec -n $namespace $pod -c analytics-assembly -- jcmd Bootstrap GC.heap_dump /tmp/$pod.hprof +kubectl exec -n $namespace $pod -c analytics-assembly -- gzip /tmp/$pod.hprof +kubectl -n $namespace cp -c analytics-assembly $pod:/tmp/$pod.hprof.gz $pod.hprof.gz + +kubectl exec -n $namespace $pod -c analytics-assembly -it -- /bin/bash \ No newline at end of file diff --git a/configs/sumo/scripts/helm-out b/configs/sumo/scripts/helm-out new file mode 100755 index 0000000..662745d --- /dev/null +++ b/configs/sumo/scripts/helm-out @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +name=${1:-new} +dep=${2:-stag} + +cmd="helm template --values test_values.yaml --values values.yaml --values values-$dep.yaml" + +if [ ! -z "values-$dep.auto.yaml"]; then + cmd="$cmd --values values-$dep.auto.yaml" +fi + +cmd="$cmd --debug . > tmp/$name.yml" +echo $cmd + +$cmd + +if [ $name == "new" ]; + + if [ ! -z "tmp/old.yml" ]; then + diff tmp/original.yml tmp/new.yml > tmp/diff + fi + +fi \ No newline at end of file diff --git a/configs/sumo/scripts/identify-perm b/configs/sumo/scripts/identify-perm new file mode 100755 index 0000000..b420d15 --- /dev/null +++ b/configs/sumo/scripts/identify-perm @@ -0,0 +1,9 @@ +#!/bin/bash + +TF_LOG=trace terraform apply --auto-approve &> apply.log +TF_LOG=trace terraform destroy --auto-approve &> destroy.log + +cat destroy.log | grep "DEBUG: Request" | sed -nr 's/^.*Request (.*) Details:/\1/p' | sort | uniq > all.perms +cat apply.log | grep "DEBUG: Request" | sed -nr 's/^.*Request (.*) Details:/\1/p' | sort | uniq >> all.perms +cat all.perms | sort | uniq -u > required.perms +rm all.perms \ No newline at end of file diff --git a/configs/sumo/scripts/mount-dev b/configs/sumo/scripts/mount-dev new file mode 100755 index 0000000..76971a8 --- /dev/null +++ b/configs/sumo/scripts/mount-dev @@ -0,0 +1,6 @@ +#/bin/zsh +set -e + +dev encrypted_disk mount +dev provision ssh_keys +ssh-add /Volumes/Encrypted/keys/prod_rsa diff --git a/configs/sumo/scripts/open-aws b/configs/sumo/scripts/open-aws new file mode 100755 index 0000000..803024a --- /dev/null +++ b/configs/sumo/scripts/open-aws @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +dev hcvault login +dev hcvault console -o diff --git a/configs/sumo/scripts/regex b/configs/sumo/scripts/regex new file mode 100755 index 0000000..7a90dbd --- /dev/null +++ b/configs/sumo/scripts/regex @@ -0,0 +1,3 @@ +#!/bin/bash + +gawk 'match($0,/'$1'/, ary) {print ary['${2:-'1'}']}' \ No newline at end of file diff --git a/configs/sumo/scripts/sagemaker b/configs/sumo/scripts/sagemaker new file mode 100755 index 0000000..2f4519a --- /dev/null +++ b/configs/sumo/scripts/sagemaker @@ -0,0 +1,3 @@ +#!/bin/bash + +eks us2 analytics-sagemaker ${@:1} \ No newline at end of file diff --git a/configs/sumo/scripts/update-aws-cred b/configs/sumo/scripts/update-aws-cred new file mode 100755 index 0000000..bdae9d4 --- /dev/null +++ b/configs/sumo/scripts/update-aws-cred @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +cred_file=${1} +aws_cred_file=$HOME/.aws/credentials + +source $HOME/$cred_file.sh + +if [ -z "${AWS_ACCESS_KEY_ID}" ]; then + echo "Missing access key"; + exit 1 +fi +if [ -z "${AWS_SECRET_ACCESS_KEY}" ]; then + echo "Missing AWS_SECRET_ACCESS_KEY"; + exit 1 +fi +if [ -z "${AWS_SESSION_TOKEN}" ]; then + echo "Missing AWS_SESSION_TOKEN"; + exit 1 +fi + +echo "Updating $aws_cred_file ..." +echo "[default]" > $aws_cred_file +echo "aws_access_key_id = $AWS_ACCESS_KEY_ID" >> $aws_cred_file +echo "aws_secret_access_key = $AWS_SECRET_ACCESS_KEY" >> $aws_cred_file +echo "aws_session_token=$AWS_SESSION_TOKEN" >> $aws_cred_file + +unset AWS_ACCESS_KEY_ID +unset AWS_SECRET_ACCESS_KEY +unset AWS_SESSION_TOKEN + +echo `aws sts get-caller-identity` diff --git a/configs/sumo/scripts/update-dsh-knobs b/configs/sumo/scripts/update-dsh-knobs new file mode 100755 index 0000000..371f05c --- /dev/null +++ b/configs/sumo/scripts/update-dsh-knobs @@ -0,0 +1,8 @@ +#!/bin/bash + +echo "<<< ◉ Updating dsh ⦿ >>>" +cd ~/git/sumo && git fetch origin master:master && git merge master && git status + +echo "<<< ◉ Updatig knobs ⦿ >>>" +cd ~/git/knobs && git fetch origin master:master && git merge master && git status + diff --git a/flake.nix b/flake.nix index 90e5649..d309016 100644 --- a/flake.nix +++ b/flake.nix @@ -45,7 +45,6 @@ hostName = "fwbook"; home-manager = { enable = true; - mode = "nixos"; version = homeMangerVersion; }; desktop = { @@ -73,5 +72,24 @@ ]; }; }; + darwinConfigurations."asosnovsky-mac" = lib.makeDarwinModule { + user = user; + systemStateVersion = 4; + system = "aarch64-darwin"; + hostName = "asosnovsky-mac"; + home-manager = { + enable = true; + version = homeMangerVersion; + }; + desktop = { + enable = false; + }; + os = { + enable = false; + }; + configuration = (import ./hosts/asosnovsky-mac.nix { + user = user; + }); + }; }; } diff --git a/hosts/asosnovsky-mac.nix b/hosts/asosnovsky-mac.nix new file mode 100644 index 0000000..6e446dd --- /dev/null +++ b/hosts/asosnovsky-mac.nix @@ -0,0 +1,47 @@ +{ user }: +{ pkgs, ... }: +let + zshSumo = builtins.filterSource (p: t: true) ../configs/sumo; + zshScripts = zshSumo + "/scripts"; + zshFunctions = zshSumo + "/functions.sh"; +in +{ + home-manager.users.${user.name} = { + home = { + packages = with pkgs; [ + coreutils-prefixed + awscli2 + ]; + shellAliases = { + k = "kubectl"; + gradlew = "dev gradlew"; + psh = "source $HOME/git/github/psh/psh.sh"; + hcvault = "dev hcvault"; + }; + sessionVariables = { + "DOCKER_DEFAULT_PLATFORM" = "linux/amd64"; + "SUMO_HOME" = "$HOME/git/sumo"; + "_JAVA_OPTIONS" = "-Djava.awt.headless=true"; + "LC_ALL" = "en_US.UTF-8"; + "JAVA_HOME" = "/Library/Java/JavaVirtualMachines/amazon-corretto-8.jdk/Contents/Home"; + }; + sessionPath = [ + "$HOME/.devcli/bin" + "$HOME/.krew/bin" + "$HOME/git/github/paas-infrastructure/bin" + "$HOME/.krew/bin" + zshScripts + ]; + }; + programs.zsh.initExtra = '' + source ${zshFunctions} + source /dev/stdin <<< "$($HOME/.devcli/dev --init)" + complete -o default -F __start_kubectl k + source <(helm completion zsh) + complete -C '/opt/homebrew/bin/aws_completer' aws + source <(kubectl completion zsh) + test -f ~/.prada/artifactory.env && source ~/.prada/artifactory.env + ulimit -n $(sysctl -n kern.maxfilesperproc) + ''; + }; +}