From 5f5856bb1567147edf6db0961efdfd8a7fab830c Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Mon, 26 Apr 2021 00:49:01 +0530 Subject: [PATCH 01/20] kubernetes HA initial commit --- kubernetes/ha/Vagrantfile | 72 ++++++++++++++++++++++++ kubernetes/ha/config.yaml | 26 +++++++++ kubernetes/ha/script/bootstrap.sh | 57 +++++++++++++++++++ kubernetes/ha/script/bootstrap_ha.sh | 46 +++++++++++++++ kubernetes/ha/script/bootstrap_master.sh | 30 ++++++++++ 5 files changed, 231 insertions(+) create mode 100644 kubernetes/ha/Vagrantfile create mode 100644 kubernetes/ha/config.yaml create mode 100644 kubernetes/ha/script/bootstrap.sh create mode 100644 kubernetes/ha/script/bootstrap_ha.sh create mode 100644 kubernetes/ha/script/bootstrap_master.sh diff --git a/kubernetes/ha/Vagrantfile b/kubernetes/ha/Vagrantfile new file mode 100644 index 0000000..d8838fa --- /dev/null +++ b/kubernetes/ha/Vagrantfile @@ -0,0 +1,72 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require 'yaml' +k8s = YAML.load_file(File.join(File.dirname(__FILE__), 'config.yaml')) +ENV["LC_ALL"] = "en_US.UTF-8" + +$msg = < /etc/sysctl.d/k8s.conf +net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1 +net.ipv4.ip_forward = 1 +EOF +sysctl --system + +# Disable all memory swaps to increase performance. +sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab +swapoff -a + +apt-get update +apt-get install -y apt-transport-https ca-certificates curl wget zip unzip vim git gnupg lsb-release software-properties-common telnet +curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg +echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y kubelet kubeadm kubectl docker-ce docker-ce-cli containerd.io +apt-mark hold kubelet kubeadm kubectl +usermod -aG docker ${1} + +cat <> $HOME/.bash_profile' +# chown ${1} /etc/kubernetes/admin.conf +# echo "export KUBEADM_JOIN=\"${join_command}\"" >> /home/${1}/.bash_profile + +# su ${1} -c "kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml" From e2181a36019de1189a6dc2a7480cab8fea3ff692 Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Mon, 26 Apr 2021 12:00:20 +0530 Subject: [PATCH 02/20] cfssl certificates --- kubernetes/ha/Vagrantfile | 123 ++++++++++++++++++++++++++- kubernetes/ha/script/bootstrap_ha.sh | 79 ++++++++++++++++- 2 files changed, 200 insertions(+), 2 deletions(-) diff --git a/kubernetes/ha/Vagrantfile b/kubernetes/ha/Vagrantfile index d8838fa..1ce2794 100644 --- a/kubernetes/ha/Vagrantfile +++ b/kubernetes/ha/Vagrantfile @@ -18,7 +18,6 @@ MSG Vagrant.configure(k8s['api_version']) do |config| config.vm.define "#{k8s['cluster']['ha']}" do |subconfig| - subconfig.vm.post_up_message = $msg subconfig.vm.box = k8s['image'] subconfig.vm.box_check_update = false @@ -67,6 +66,128 @@ Vagrant.configure(k8s['api_version']) do |config| end end +Vagrant.configure(k8s['api_version']) do |config| + config.vm.define "#{k8s['cluster']['master']}" do |subconfig| + subconfig.vm.post_up_message = $msg + subconfig.vm.box = k8s['image'] + subconfig.vm.box_check_update = false + + subconfig.vm.hostname = "#{k8s['cluster']['master']}" + subconfig.vm.network :private_network, ip: "#{k8s['ip_part']}.10" + # Hostfile :: Master node + subconfig.vm.provision "master-hostfile", type: "shell" do |mhf| + mhf.inline = <<-SHELL + echo -e "127.0.0.1\t$2" | tee -a /etc/hosts; echo -e "$1\t$2" | tee -a /etc/hosts + SHELL + mhf.args = ["#{k8s['ip_part']}.10", "#{k8s['cluster']['master']}"] + end + # Hostfile :: Worker node + subconfig.vm.provision "Update hostfile and authorized_keys", type: "shell" do |whu| + whu.inline = <<-SHELL + for i in $(eval echo {1..$2}); do + echo -e "${3}.$((10 + $i))\t#{k8s['cluster']['node']}-${i}" | tee -a /etc/hosts + done + SHELL + whu.args = ["#{k8s['user']}", "#{k8s['resources']['node']['count']}", "#{k8s['ip_part']}"] + end + subconfig.vm.provider "virtualbox" do |vb| + vb.memory = k8s['resources']['master']['memory'] + vb.cpus = k8s['resources']['master']['cpus'] + end + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| + reboot.privileged = true + reboot.inline = <<-SHELL + echo "----------------------------------|| Reboot to load all config" + SHELL + reboot.reboot = true + end + + subconfig.vm.provision "#{k8s['cluster']['master']}-setup", type: "shell" do |mns| + mns.path = "script/bootstrap_master.sh" + mns.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "10"] + end + + subconfig.trigger.after :up do |trigger_local| + trigger_local.run = {inline: "/bin/bash -c 'vagrant ssh --no-tty -c \"cat /etc/kubernetes/admin.conf\" #{k8s['cluster']['master']} > admin.conf && rm -f \${HOME}/.kube/config 2>/dev/null; mkdir -p \${HOME}/.kube; cp -i admin.conf \${HOME}/.kube/config; rm -f admin.conf'"} + end + end + + (1..k8s['resources']['node']['count']).each do |i| + config.vm.define "#{k8s['cluster']['node']}-#{i}" do |subconfig| + subconfig.vm.box = k8s['image'] + + subconfig.vm.hostname = "#{k8s['cluster']['node']}-#{i}" + subconfig.vm.network :private_network, ip: "#{k8s['ip_part']}.#{i + 10}" + + # Hostfile :: Master node + subconfig.vm.provision "master-hostfile", type: "shell" do |s| + s.inline = <<-SHELL + echo -e "$1\t$2" | tee -a /etc/hosts + ufw allow 10250/tcp + ufw allow 10251/tcp + ufw allow 10255/tcp + ufw allow 30000:32767/tcp + ufw reload + SHELL + s.args = ["#{k8s['ip_part']}.10", "#{k8s['cluster']['master']}"] + end + # Hostfile :: Worker node + (1..k8s['resources']['node']['count']).each do |j| + if i != j + subconfig.vm.provision "other-worker-hostfile", type: "shell" do |supdate| + supdate.inline = <<-SHELL + echo -e "$1\t$2" | tee -a /etc/hosts + SHELL + supdate.args = ["#{k8s['ip_part']}.#{10 + j}", "#{k8s['cluster']['node']}-#{j}", "#{k8s['user']}", "#{i}"] + end + else + subconfig.vm.provision "self-worker-hostfile", type: "shell" do |supdate| + supdate.inline = <<-SHELL + echo -e "127.0.0.1\t$2" | tee -a /etc/hosts; echo -e "$1\t$2" | tee -a /etc/hosts + SHELL + supdate.args = ["#{k8s['ip_part']}.#{10 + j}", "#{k8s['cluster']['node']}-#{j}", "#{k8s['user']}", "#{i}"] + end + end + end + + subconfig.vm.provider "virtualbox" do |vb| + vb.memory = k8s['resources']['node']['memory'] + vb.cpus = k8s['resources']['node']['cpus'] + end + + subconfig.trigger.after :up do |trigger_local| + trigger_local.run = {inline: "/bin/bash -c 'wpub_key=$(vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['node']}-#{i}) && vagrant ssh --no-tty -c \"echo \${wpub_key} >> /home/#{k8s['user']}/.ssh/authorized_keys\" #{k8s['cluster']['master']}; mpub_key=$(vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['master']}) && vagrant ssh --no-tty -c \"echo \${mpub_key} >> /home/#{k8s['user']}/.ssh/authorized_keys\" #{k8s['cluster']['node']}-#{i}'"} + end + + subconfig.trigger.after :up do |trigger_remote| + trigger_remote.run_remote = {inline: <<-SHELL + kube_join=\$(echo "ssh #{k8s['user']}@#{k8s['cluster']['master']} -o StrictHostKeyChecking=no '( cat /home/#{k8s['user']}/.bash_profile | grep KUBEADM_JOIN)'" | su - #{k8s['user']}) + kube_join=\$(echo ${kube_join} | awk -F'"' '{print \$2}') + echo "sudo $kube_join" | su - #{k8s['user']} + echo "scp -o StrictHostKeyChecking=no #{k8s['user']}@#{k8s['cluster']['master']}:/etc/kubernetes/admin.conf /home/#{k8s['user']}/" | su - #{k8s['user']} + echo "mkdir -p /home/#{k8s['user']}/.kube" | su - #{k8s['user']} + echo "cp -i /home/#{k8s['user']}/admin.conf /home/#{k8s['user']}/.kube/config" | su - #{k8s['user']} + echo "sudo chown #{k8s['user']}:#{k8s['user']} -R /home/#{k8s['user']}/.kube" | su - #{k8s['user']} + echo "kubectl label nodes #{k8s['cluster']['node']}-#{i} kubernetes.io/role=#{k8s['cluster']['node']}-#{i}" | su - #{k8s['user']} + SHELL + } + end + + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| + reboot.privileged = true + reboot.inline = <<-SHELL + echo "----------------------------------|| Reboot to load all config" + SHELL + reboot.reboot = true + end + end + end + + config.vm.provision "vm-setup", type: "shell" do |vms| + vms.path = "script/bootstrap.sh" + vms.args = ["#{k8s['user']}"] + end +end diff --git a/kubernetes/ha/script/bootstrap_ha.sh b/kubernetes/ha/script/bootstrap_ha.sh index fe45cf7..2231c8c 100644 --- a/kubernetes/ha/script/bootstrap_ha.sh +++ b/kubernetes/ha/script/bootstrap_ha.sh @@ -34,7 +34,7 @@ systemctl enable --now haproxy modprobe overlay modprobe br_netfilter -echo "ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa << ca-config.json < ca-csr.json < kubernetes-csr.json < Date: Thu, 29 Apr 2021 16:18:22 +0530 Subject: [PATCH 03/20] ha: download ssh keys --- kubernetes/poc/Vagrantfile | 38 ++++++++++++ kubernetes/poc/config.yaml | 24 ++++++++ kubernetes/poc/lib/master.rb | 52 ++++++++++++++++ kubernetes/poc/lib/node.rb | 73 +++++++++++++++++++++++ kubernetes/poc/script/bootstrap.sh | 53 ++++++++++++++++ kubernetes/poc/script/bootstrap_master.sh | 30 ++++++++++ 6 files changed, 270 insertions(+) create mode 100644 kubernetes/poc/Vagrantfile create mode 100644 kubernetes/poc/config.yaml create mode 100644 kubernetes/poc/lib/master.rb create mode 100644 kubernetes/poc/lib/node.rb create mode 100644 kubernetes/poc/script/bootstrap.sh create mode 100644 kubernetes/poc/script/bootstrap_master.sh diff --git a/kubernetes/poc/Vagrantfile b/kubernetes/poc/Vagrantfile new file mode 100644 index 0000000..16aa3e3 --- /dev/null +++ b/kubernetes/poc/Vagrantfile @@ -0,0 +1,38 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require 'yaml' +k8s = YAML.load_file(File.join(File.dirname(__FILE__), 'config.yaml')) +ENV["LC_ALL"] = "en_US.UTF-8" + +$msg = < tmp/#{k8s['cluster']['master']}-#{i}.pub'"} + end +end \ No newline at end of file diff --git a/kubernetes/poc/lib/node.rb b/kubernetes/poc/lib/node.rb new file mode 100644 index 0000000..5103498 --- /dev/null +++ b/kubernetes/poc/lib/node.rb @@ -0,0 +1,73 @@ +config.vm.define "#{k8s['cluster']['node']}-#{i}" do |subconfig| + subconfig.vm.box = k8s['image'] + + subconfig.vm.hostname = "#{k8s['cluster']['node']}-#{i}" + subconfig.vm.network :private_network, ip: "#{k8s['ip_part']}.#{i + k8s['resources']['node']['ip_prefix']}" + + # Hostfile :: Master node + subconfig.vm.provision "Load Balancer hostfile update", type: "shell" do |lb| + lb.inline = <<-SHELL + echo -e "127.0.0.1\t$1" | tee -a /etc/hosts + SHELL + lb.args = ["#{k8s['cluster']['node']}"] + end + subconfig.vm.provision "Master and Worker node hostfile update", type: "shell" do |cluster| + cluster.inline = <<-SHELL + # master + for i in $(eval echo {1..#{k8s['resources']['master']['count']}}); do + echo -e "${1}.$((#{k8s['resources']['master']['ip_prefix']} + $i))\t#{k8s['cluster']['master']}-${i}" | tee -a /etc/hosts + done + + # worker + for i in $(eval echo {1..#{k8s['resources']['node']['count']}}); do + echo -e "${1}.$((#{k8s['resources']['node']['ip_prefix']} + $i))\t#{k8s['cluster']['node']}-${i}" | tee -a /etc/hosts + done + SHELL + cluster.args = ["#{k8s['ip_part']}"] + end + + subconfig.vm.provider "virtualbox" do |vb| + vb.memory = k8s['resources']['node']['memory'] + vb.cpus = k8s['resources']['node']['cpus'] + end + + # subconfig.trigger.after :up do |trigger_local| + # trigger_local.run = {inline: "/bin/bash -c 'wpub_key=$(vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['node']}-#{i}) && vagrant ssh --no-tty -c \"echo \${wpub_key} >> /home/#{k8s['user']}/.ssh/authorized_keys\" #{k8s['cluster']['master']}; mpub_key=$(vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['master']}) && vagrant ssh --no-tty -c \"echo \${mpub_key} >> /home/#{k8s['user']}/.ssh/authorized_keys\" #{k8s['cluster']['node']}-#{i}'"} + # end + + subconfig.trigger.after :up do |trigger_local| + trigger_local.run = {inline: "/bin/bash -c 'vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['master']}-#{i} > tmp/#{k8s['cluster']['master']}-#{i}.pub'"} + end + + subconfig.vm.provision "firewall update", type: "shell" do |s| + s.inline = <<-SHELL + ufw allow 10250/tcp + ufw allow 10251/tcp + ufw allow 10255/tcp + ufw allow 30000:32767/tcp + ufw reload + SHELL + end + + # subconfig.trigger.after :up do |trigger_remote| + # trigger_remote.run_remote = {inline: <<-SHELL + # kube_join=\$(echo "ssh #{k8s['user']}@#{k8s['cluster']['master']} -o StrictHostKeyChecking=no '( cat /home/#{k8s['user']}/.bash_profile | grep KUBEADM_JOIN)'" | su - #{k8s['user']}) + # kube_join=\$(echo ${kube_join} | awk -F'"' '{print \$2}') + # echo "sudo $kube_join" | su - #{k8s['user']} + # echo "scp -o StrictHostKeyChecking=no #{k8s['user']}@#{k8s['cluster']['master']}:/etc/kubernetes/admin.conf /home/#{k8s['user']}/" | su - #{k8s['user']} + # echo "mkdir -p /home/#{k8s['user']}/.kube" | su - #{k8s['user']} + # echo "cp -i /home/#{k8s['user']}/admin.conf /home/#{k8s['user']}/.kube/config" | su - #{k8s['user']} + # echo "sudo chown #{k8s['user']}:#{k8s['user']} -R /home/#{k8s['user']}/.kube" | su - #{k8s['user']} + # echo "kubectl label nodes #{k8s['cluster']['node']}-#{i} kubernetes.io/role=#{k8s['cluster']['node']}-#{i}" | su - #{k8s['user']} + # SHELL + # } + # end + + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| + reboot.privileged = true + reboot.inline = <<-SHELL + echo "----------------------------------|| Reboot to load all config" + SHELL + reboot.reboot = true + end +end \ No newline at end of file diff --git a/kubernetes/poc/script/bootstrap.sh b/kubernetes/poc/script/bootstrap.sh new file mode 100644 index 0000000..43e6029 --- /dev/null +++ b/kubernetes/poc/script/bootstrap.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +cat < /etc/sysctl.d/k8s.conf +net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1 +net.ipv4.ip_forward = 1 +EOF +sysctl --system + +# Disable all memory swaps to increase performance. +sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab +swapoff -a + +apt-get update +apt-get install -y apt-transport-https ca-certificates curl wget zip unzip vim git gnupg lsb-release software-properties-common telnet +curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io +usermod -aG docker ${1} + +cat <> $HOME/.bash_profile' +# chown ${1} /etc/kubernetes/admin.conf +# echo "export KUBEADM_JOIN=\"${join_command}\"" >> /home/${1}/.bash_profile + +# su ${1} -c "kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml" From 1cd609f9bcec384dfe7642db72e0c0caef5336ed Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Thu, 29 Apr 2021 22:37:41 +0530 Subject: [PATCH 04/20] ha: fix for ssh keys distribution --- kubernetes/poc/Vagrantfile | 47 +++++++++++++++++++--- kubernetes/poc/lib/master.rb | 34 ++++++++-------- kubernetes/poc/lib/node.rb | 36 ++++------------- kubernetes/poc/script/bootstrap copy.sh | 53 +++++++++++++++++++++++++ kubernetes/poc/script/bootstrap.sh | 47 ---------------------- 5 files changed, 118 insertions(+), 99 deletions(-) create mode 100644 kubernetes/poc/script/bootstrap copy.sh diff --git a/kubernetes/poc/Vagrantfile b/kubernetes/poc/Vagrantfile index 16aa3e3..b7cf7a7 100644 --- a/kubernetes/poc/Vagrantfile +++ b/kubernetes/poc/Vagrantfile @@ -2,10 +2,12 @@ # vi: set ft=ruby : require 'yaml' +require 'open3' + k8s = YAML.load_file(File.join(File.dirname(__FILE__), 'config.yaml')) ENV["LC_ALL"] = "en_US.UTF-8" -$msg = <> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{n}") + end + + 1.step(k8s['resources']['node']['count']) do |e| + system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{e}") + end + end + + 1.step(k8s['resources']['node']['count']) do |m| + wpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['node'] + "-#{m}") + + 1.step(k8s['resources']['node']['count']) do |n| + next if m == n + system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{n}") + end + + 1.step(k8s['resources']['master']['count']) do |e| + system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{e}") + end + end + end + end end diff --git a/kubernetes/poc/lib/master.rb b/kubernetes/poc/lib/master.rb index 6aacf29..a673aed 100644 --- a/kubernetes/poc/lib/master.rb +++ b/kubernetes/poc/lib/master.rb @@ -1,5 +1,5 @@ config.vm.define "#{k8s['cluster']['master']}-#{i}" do |subconfig| - subconfig.vm.post_up_message = $msg + # subconfig.vm.post_up_message = $msg subconfig.vm.box = k8s['image'] subconfig.vm.box_check_update = false @@ -11,7 +11,7 @@ lb.inline = <<-SHELL echo -e "127.0.0.1\t$1" | tee -a /etc/hosts SHELL - lb.args = ["#{k8s['cluster']['master']}"] + lb.args = ["#{k8s['cluster']['master']}-#{i}"] end subconfig.vm.provision "Master and Worker node hostfile update", type: "shell" do |cluster| cluster.inline = <<-SHELL @@ -29,24 +29,22 @@ end subconfig.vm.provider "virtualbox" do |vb| + vb.name = "#{k8s['cluster']['master']}-#{i}" vb.memory = k8s['resources']['master']['memory'] vb.cpus = k8s['resources']['master']['cpus'] + vb.gui = false end - subconfig.vm.provision "Restart VM", type: "shell" do |reboot| - reboot.privileged = true - reboot.inline = <<-SHELL - echo "----------------------------------|| Reboot to load all config" - SHELL - reboot.reboot = true - end + # subconfig.vm.provision "Restart VM", type: "shell" do |reboot| + # reboot.privileged = true + # reboot.inline = <<-SHELL + # echo "----------------------------------|| Reboot to load all config" + # SHELL + # reboot.reboot = true + # end - subconfig.vm.provision "#{k8s['cluster']['master']}-setup", type: "shell" do |mns| - mns.path = "script/bootstrap_master.sh" - mns.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "10"] - end - - subconfig.trigger.after :up do |trigger_local| - trigger_local.run = {inline: "/bin/bash -c 'vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['master']}-#{i} > tmp/#{k8s['cluster']['master']}-#{i}.pub'"} - end -end \ No newline at end of file + # subconfig.vm.provision "#{k8s['cluster']['master']}-setup", type: "shell" do |mns| + # mns.path = "script/bootstrap_master.sh" + # mns.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "10"] + # end +end diff --git a/kubernetes/poc/lib/node.rb b/kubernetes/poc/lib/node.rb index 5103498..4f97cd0 100644 --- a/kubernetes/poc/lib/node.rb +++ b/kubernetes/poc/lib/node.rb @@ -29,14 +29,8 @@ subconfig.vm.provider "virtualbox" do |vb| vb.memory = k8s['resources']['node']['memory'] vb.cpus = k8s['resources']['node']['cpus'] - end - - # subconfig.trigger.after :up do |trigger_local| - # trigger_local.run = {inline: "/bin/bash -c 'wpub_key=$(vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['node']}-#{i}) && vagrant ssh --no-tty -c \"echo \${wpub_key} >> /home/#{k8s['user']}/.ssh/authorized_keys\" #{k8s['cluster']['master']}; mpub_key=$(vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['master']}) && vagrant ssh --no-tty -c \"echo \${mpub_key} >> /home/#{k8s['user']}/.ssh/authorized_keys\" #{k8s['cluster']['node']}-#{i}'"} - # end - - subconfig.trigger.after :up do |trigger_local| - trigger_local.run = {inline: "/bin/bash -c 'vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['master']}-#{i} > tmp/#{k8s['cluster']['master']}-#{i}.pub'"} + vb.name = "#{k8s['cluster']['node']}-#{i}" + vb.gui = false end subconfig.vm.provision "firewall update", type: "shell" do |s| @@ -49,25 +43,11 @@ SHELL end - # subconfig.trigger.after :up do |trigger_remote| - # trigger_remote.run_remote = {inline: <<-SHELL - # kube_join=\$(echo "ssh #{k8s['user']}@#{k8s['cluster']['master']} -o StrictHostKeyChecking=no '( cat /home/#{k8s['user']}/.bash_profile | grep KUBEADM_JOIN)'" | su - #{k8s['user']}) - # kube_join=\$(echo ${kube_join} | awk -F'"' '{print \$2}') - # echo "sudo $kube_join" | su - #{k8s['user']} - # echo "scp -o StrictHostKeyChecking=no #{k8s['user']}@#{k8s['cluster']['master']}:/etc/kubernetes/admin.conf /home/#{k8s['user']}/" | su - #{k8s['user']} - # echo "mkdir -p /home/#{k8s['user']}/.kube" | su - #{k8s['user']} - # echo "cp -i /home/#{k8s['user']}/admin.conf /home/#{k8s['user']}/.kube/config" | su - #{k8s['user']} - # echo "sudo chown #{k8s['user']}:#{k8s['user']} -R /home/#{k8s['user']}/.kube" | su - #{k8s['user']} - # echo "kubectl label nodes #{k8s['cluster']['node']}-#{i} kubernetes.io/role=#{k8s['cluster']['node']}-#{i}" | su - #{k8s['user']} - # SHELL - # } + # subconfig.vm.provision "Restart VM", type: "shell" do |reboot| + # reboot.privileged = true + # reboot.inline = <<-SHELL + # echo "----------------------------------|| Reboot to load all config" + # SHELL + # reboot.reboot = true # end - - subconfig.vm.provision "Restart VM", type: "shell" do |reboot| - reboot.privileged = true - reboot.inline = <<-SHELL - echo "----------------------------------|| Reboot to load all config" - SHELL - reboot.reboot = true - end end \ No newline at end of file diff --git a/kubernetes/poc/script/bootstrap copy.sh b/kubernetes/poc/script/bootstrap copy.sh new file mode 100644 index 0000000..43e6029 --- /dev/null +++ b/kubernetes/poc/script/bootstrap copy.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +cat < /etc/sysctl.d/k8s.conf +net.bridge.bridge-nf-call-ip6tables = 1 +net.bridge.bridge-nf-call-iptables = 1 +net.ipv4.ip_forward = 1 +EOF +sysctl --system + +# Disable all memory swaps to increase performance. +sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab +swapoff -a + +apt-get update +apt-get install -y apt-transport-https ca-certificates curl wget zip unzip vim git gnupg lsb-release software-properties-common telnet +curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce docker-ce-cli containerd.io +usermod -aG docker ${1} + +cat < /etc/sysctl.d/k8s.conf -net.bridge.bridge-nf-call-ip6tables = 1 -net.bridge.bridge-nf-call-iptables = 1 -net.ipv4.ip_forward = 1 -EOF -sysctl --system - -# Disable all memory swaps to increase performance. -sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab -swapoff -a - -apt-get update -apt-get install -y apt-transport-https ca-certificates curl wget zip unzip vim git gnupg lsb-release software-properties-common telnet -curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -apt-get update -apt-get install -y docker-ce docker-ce-cli containerd.io -usermod -aG docker ${1} - -cat < Date: Thu, 29 Apr 2021 22:44:57 +0530 Subject: [PATCH 05/20] ha: clean workspace --- kubernetes/ha/Vagrantfile | 206 +++++----------------- kubernetes/ha/config.yaml | 14 +- kubernetes/{poc => ha}/lib/master.rb | 22 +-- kubernetes/{poc => ha}/lib/node.rb | 14 +- kubernetes/ha/script/bootstrap.sh | 8 +- kubernetes/ha/script/bootstrap_ha.sh | 123 ------------- kubernetes/poc/Vagrantfile | 73 -------- kubernetes/poc/config.yaml | 24 --- kubernetes/poc/script/bootstrap copy.sh | 53 ------ kubernetes/poc/script/bootstrap.sh | 6 - kubernetes/poc/script/bootstrap_master.sh | 30 ---- 11 files changed, 69 insertions(+), 504 deletions(-) rename kubernetes/{poc => ha}/lib/master.rb (76%) rename kubernetes/{poc => ha}/lib/node.rb (86%) delete mode 100644 kubernetes/ha/script/bootstrap_ha.sh delete mode 100644 kubernetes/poc/Vagrantfile delete mode 100644 kubernetes/poc/config.yaml delete mode 100644 kubernetes/poc/script/bootstrap copy.sh delete mode 100644 kubernetes/poc/script/bootstrap.sh delete mode 100644 kubernetes/poc/script/bootstrap_master.sh diff --git a/kubernetes/ha/Vagrantfile b/kubernetes/ha/Vagrantfile index 1ce2794..b7cf7a7 100644 --- a/kubernetes/ha/Vagrantfile +++ b/kubernetes/ha/Vagrantfile @@ -2,10 +2,12 @@ # vi: set ft=ruby : require 'yaml' +require 'open3' + k8s = YAML.load_file(File.join(File.dirname(__FILE__), 'config.yaml')) ENV["LC_ALL"] = "en_US.UTF-8" -$msg = < admin.conf && rm -f \${HOME}/.kube/config 2>/dev/null; mkdir -p \${HOME}/.kube; cp -i admin.conf \${HOME}/.kube/config; rm -f admin.conf'"} + config.vm.boot_timeout = 600 + + (1..k8s['resources']['master']['count']).each do |i| + if File.exist?('lib/master.rb') + eval(IO.read('lib/master.rb'), binding) end end (1..k8s['resources']['node']['count']).each do |i| - config.vm.define "#{k8s['cluster']['node']}-#{i}" do |subconfig| - subconfig.vm.box = k8s['image'] - - subconfig.vm.hostname = "#{k8s['cluster']['node']}-#{i}" - subconfig.vm.network :private_network, ip: "#{k8s['ip_part']}.#{i + 10}" - - # Hostfile :: Master node - subconfig.vm.provision "master-hostfile", type: "shell" do |s| - s.inline = <<-SHELL - echo -e "$1\t$2" | tee -a /etc/hosts - ufw allow 10250/tcp - ufw allow 10251/tcp - ufw allow 10255/tcp - ufw allow 30000:32767/tcp - ufw reload - SHELL - s.args = ["#{k8s['ip_part']}.10", "#{k8s['cluster']['master']}"] - end - # Hostfile :: Worker node - (1..k8s['resources']['node']['count']).each do |j| - if i != j - subconfig.vm.provision "other-worker-hostfile", type: "shell" do |supdate| - supdate.inline = <<-SHELL - echo -e "$1\t$2" | tee -a /etc/hosts - SHELL - supdate.args = ["#{k8s['ip_part']}.#{10 + j}", "#{k8s['cluster']['node']}-#{j}", "#{k8s['user']}", "#{i}"] - end - else - subconfig.vm.provision "self-worker-hostfile", type: "shell" do |supdate| - supdate.inline = <<-SHELL - echo -e "127.0.0.1\t$2" | tee -a /etc/hosts; echo -e "$1\t$2" | tee -a /etc/hosts - SHELL - supdate.args = ["#{k8s['ip_part']}.#{10 + j}", "#{k8s['cluster']['node']}-#{j}", "#{k8s['user']}", "#{i}"] - end - end - end - - subconfig.vm.provider "virtualbox" do |vb| - vb.memory = k8s['resources']['node']['memory'] - vb.cpus = k8s['resources']['node']['cpus'] - end - - subconfig.trigger.after :up do |trigger_local| - trigger_local.run = {inline: "/bin/bash -c 'wpub_key=$(vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['node']}-#{i}) && vagrant ssh --no-tty -c \"echo \${wpub_key} >> /home/#{k8s['user']}/.ssh/authorized_keys\" #{k8s['cluster']['master']}; mpub_key=$(vagrant ssh --no-tty -c \"cat /home/#{k8s['user']}/.ssh/id_rsa.pub\" #{k8s['cluster']['master']}) && vagrant ssh --no-tty -c \"echo \${mpub_key} >> /home/#{k8s['user']}/.ssh/authorized_keys\" #{k8s['cluster']['node']}-#{i}'"} - end - - subconfig.trigger.after :up do |trigger_remote| - trigger_remote.run_remote = {inline: <<-SHELL - kube_join=\$(echo "ssh #{k8s['user']}@#{k8s['cluster']['master']} -o StrictHostKeyChecking=no '( cat /home/#{k8s['user']}/.bash_profile | grep KUBEADM_JOIN)'" | su - #{k8s['user']}) - kube_join=\$(echo ${kube_join} | awk -F'"' '{print \$2}') - echo "sudo $kube_join" | su - #{k8s['user']} - echo "scp -o StrictHostKeyChecking=no #{k8s['user']}@#{k8s['cluster']['master']}:/etc/kubernetes/admin.conf /home/#{k8s['user']}/" | su - #{k8s['user']} - echo "mkdir -p /home/#{k8s['user']}/.kube" | su - #{k8s['user']} - echo "cp -i /home/#{k8s['user']}/admin.conf /home/#{k8s['user']}/.kube/config" | su - #{k8s['user']} - echo "sudo chown #{k8s['user']}:#{k8s['user']} -R /home/#{k8s['user']}/.kube" | su - #{k8s['user']} - echo "kubectl label nodes #{k8s['cluster']['node']}-#{i} kubernetes.io/role=#{k8s['cluster']['node']}-#{i}" | su - #{k8s['user']} - SHELL - } - end - - subconfig.vm.provision "Restart VM", type: "shell" do |reboot| - reboot.privileged = true - reboot.inline = <<-SHELL - echo "----------------------------------|| Reboot to load all config" - SHELL - reboot.reboot = true - end + if File.exist?('lib/node.rb') + eval(IO.read('lib/node.rb'), binding) end end @@ -190,4 +37,37 @@ Vagrant.configure(k8s['api_version']) do |config| vms.path = "script/bootstrap.sh" vms.args = ["#{k8s['user']}"] end + + config.trigger.after :up do |trigger| + trigger.only_on = "#{k8s['cluster']['node']}-#{k8s['resources']['node']['count']}" + trigger.info = msg + + trigger.ruby do |env,machine| + 1.step(k8s['resources']['master']['count']) do |m| + mpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['master'] + "-#{m}") + + 1.step(k8s['resources']['master']['count']) do |n| + next if m == n + system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{n}") + end + + 1.step(k8s['resources']['node']['count']) do |e| + system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{e}") + end + end + + 1.step(k8s['resources']['node']['count']) do |m| + wpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['node'] + "-#{m}") + + 1.step(k8s['resources']['node']['count']) do |n| + next if m == n + system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{n}") + end + + 1.step(k8s['resources']['master']['count']) do |e| + system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{e}") + end + end + end + end end diff --git a/kubernetes/ha/config.yaml b/kubernetes/ha/config.yaml index 9c06081..fc62315 100644 --- a/kubernetes/ha/config.yaml +++ b/kubernetes/ha/config.yaml @@ -1,26 +1,24 @@ --- api_version: "2" image: "bento/ubuntu-18.04" -ip_part: "192.160.0" +ip_part: "10.240.0" user: "vagrant" cluster: - master: "master-node" - node: "worker-node" - ha: "load-balancer" + master: "controller" + node: "worker" resources: master: - cpus: 2 + cpus: 3 memory: 2048 count: 3 + ip_prefix: 10 node: cpus: 2 memory: 2048 count: 2 - ha: - cpus: 1 - memory: 1024 + ip_prefix: 20 net: network_type: private_network diff --git a/kubernetes/poc/lib/master.rb b/kubernetes/ha/lib/master.rb similarity index 76% rename from kubernetes/poc/lib/master.rb rename to kubernetes/ha/lib/master.rb index a673aed..18c36ee 100644 --- a/kubernetes/poc/lib/master.rb +++ b/kubernetes/ha/lib/master.rb @@ -35,16 +35,16 @@ vb.gui = false end - # subconfig.vm.provision "Restart VM", type: "shell" do |reboot| - # reboot.privileged = true - # reboot.inline = <<-SHELL - # echo "----------------------------------|| Reboot to load all config" - # SHELL - # reboot.reboot = true - # end + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| + reboot.privileged = true + reboot.inline = <<-SHELL + echo "----------------------------------|| Reboot to load all config" + SHELL + reboot.reboot = true + end - # subconfig.vm.provision "#{k8s['cluster']['master']}-setup", type: "shell" do |mns| - # mns.path = "script/bootstrap_master.sh" - # mns.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "10"] - # end + subconfig.vm.provision "#{k8s['cluster']['master']}-setup", type: "shell" do |mns| + mns.path = "script/bootstrap_master.sh" + mns.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "10"] + end end diff --git a/kubernetes/poc/lib/node.rb b/kubernetes/ha/lib/node.rb similarity index 86% rename from kubernetes/poc/lib/node.rb rename to kubernetes/ha/lib/node.rb index 4f97cd0..c7edd4f 100644 --- a/kubernetes/poc/lib/node.rb +++ b/kubernetes/ha/lib/node.rb @@ -43,11 +43,11 @@ SHELL end - # subconfig.vm.provision "Restart VM", type: "shell" do |reboot| - # reboot.privileged = true - # reboot.inline = <<-SHELL - # echo "----------------------------------|| Reboot to load all config" - # SHELL - # reboot.reboot = true - # end + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| + reboot.privileged = true + reboot.inline = <<-SHELL + echo "----------------------------------|| Reboot to load all config" + SHELL + reboot.reboot = true + end end \ No newline at end of file diff --git a/kubernetes/ha/script/bootstrap.sh b/kubernetes/ha/script/bootstrap.sh index 4fa3e5a..43e6029 100644 --- a/kubernetes/ha/script/bootstrap.sh +++ b/kubernetes/ha/script/bootstrap.sh @@ -18,12 +18,10 @@ swapoff -a apt-get update apt-get install -y apt-transport-https ca-certificates curl wget zip unzip vim git gnupg lsb-release software-properties-common telnet curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg -echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" apt-get update -apt-get install -y kubelet kubeadm kubectl docker-ce docker-ce-cli containerd.io -apt-mark hold kubelet kubeadm kubectl +apt-get install -y docker-ce docker-ce-cli containerd.io usermod -aG docker ${1} cat < ca-config.json < ca-csr.json < kubernetes-csr.json <> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{n}") - end - - 1.step(k8s['resources']['node']['count']) do |e| - system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{e}") - end - end - - 1.step(k8s['resources']['node']['count']) do |m| - wpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['node'] + "-#{m}") - - 1.step(k8s['resources']['node']['count']) do |n| - next if m == n - system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{n}") - end - - 1.step(k8s['resources']['master']['count']) do |e| - system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{e}") - end - end - end - end -end diff --git a/kubernetes/poc/config.yaml b/kubernetes/poc/config.yaml deleted file mode 100644 index fc62315..0000000 --- a/kubernetes/poc/config.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -api_version: "2" -image: "bento/ubuntu-18.04" -ip_part: "10.240.0" -user: "vagrant" - -cluster: - master: "controller" - node: "worker" - -resources: - master: - cpus: 3 - memory: 2048 - count: 3 - ip_prefix: 10 - node: - cpus: 2 - memory: 2048 - count: 2 - ip_prefix: 20 - -net: - network_type: private_network diff --git a/kubernetes/poc/script/bootstrap copy.sh b/kubernetes/poc/script/bootstrap copy.sh deleted file mode 100644 index 43e6029..0000000 --- a/kubernetes/poc/script/bootstrap copy.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -cat < /etc/sysctl.d/k8s.conf -net.bridge.bridge-nf-call-ip6tables = 1 -net.bridge.bridge-nf-call-iptables = 1 -net.ipv4.ip_forward = 1 -EOF -sysctl --system - -# Disable all memory swaps to increase performance. -sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab -swapoff -a - -apt-get update -apt-get install -y apt-transport-https ca-certificates curl wget zip unzip vim git gnupg lsb-release software-properties-common telnet -curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -apt-get update -apt-get install -y docker-ce docker-ce-cli containerd.io -usermod -aG docker ${1} - -cat <> $HOME/.bash_profile' -# chown ${1} /etc/kubernetes/admin.conf -# echo "export KUBEADM_JOIN=\"${join_command}\"" >> /home/${1}/.bash_profile - -# su ${1} -c "kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml" From 8b0095e84f600a461a9f4786094e64645d1730fb Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Fri, 30 Apr 2021 13:42:03 +0530 Subject: [PATCH 06/20] ha: update config --- kubernetes/ha/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ha/config.yaml b/kubernetes/ha/config.yaml index fc62315..bec3196 100644 --- a/kubernetes/ha/config.yaml +++ b/kubernetes/ha/config.yaml @@ -10,7 +10,7 @@ cluster: resources: master: - cpus: 3 + cpus: 2 memory: 2048 count: 3 ip_prefix: 10 From da6842d25ae8f3ecafd2f3af33a1d548b7842b19 Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Sat, 1 May 2021 13:30:13 +0530 Subject: [PATCH 07/20] ha: vagrantfile fix for hostentry for loadbalancer --- kubernetes/ha/Vagrantfile | 46 ++++++-------------------- kubernetes/ha/config.yaml | 5 +++ kubernetes/ha/lib/ha.rb | 49 ++++++++++++++++++++++++++++ kubernetes/ha/lib/master.rb | 9 +++-- kubernetes/ha/lib/node.rb | 11 +++++-- kubernetes/ha/lib/ssh.rb | 40 +++++++++++++++++++++++ kubernetes/ha/script/bootstrap_ha.sh | 43 ++++++++++++++++++++++++ 7 files changed, 162 insertions(+), 41 deletions(-) create mode 100644 kubernetes/ha/lib/ha.rb create mode 100644 kubernetes/ha/lib/ssh.rb create mode 100644 kubernetes/ha/script/bootstrap_ha.sh diff --git a/kubernetes/ha/Vagrantfile b/kubernetes/ha/Vagrantfile index b7cf7a7..e2175ce 100644 --- a/kubernetes/ha/Vagrantfile +++ b/kubernetes/ha/Vagrantfile @@ -21,53 +21,27 @@ MSG Vagrant.configure(k8s['api_version']) do |config| config.vm.boot_timeout = 600 + # Load Balancer vm + if File.exist?('lib/ha.rb') + eval(IO.read('lib/ha.rb'), binding) + end + + # Kubernetes Controller cluster (1..k8s['resources']['master']['count']).each do |i| if File.exist?('lib/master.rb') eval(IO.read('lib/master.rb'), binding) end end + # Kubernetes Worker cluster (1..k8s['resources']['node']['count']).each do |i| if File.exist?('lib/node.rb') eval(IO.read('lib/node.rb'), binding) end end - config.vm.provision "vm-setup", type: "shell" do |vms| - vms.path = "script/bootstrap.sh" - vms.args = ["#{k8s['user']}"] + # Exchange ssh keys to access each other, expect HA can access each of vm but not other vm cannot to access HA directly. + if File.exist?('lib/ssh.rb') + eval(IO.read('lib/ssh.rb'), binding) end - - config.trigger.after :up do |trigger| - trigger.only_on = "#{k8s['cluster']['node']}-#{k8s['resources']['node']['count']}" - trigger.info = msg - - trigger.ruby do |env,machine| - 1.step(k8s['resources']['master']['count']) do |m| - mpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['master'] + "-#{m}") - - 1.step(k8s['resources']['master']['count']) do |n| - next if m == n - system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{n}") - end - - 1.step(k8s['resources']['node']['count']) do |e| - system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{e}") - end - end - - 1.step(k8s['resources']['node']['count']) do |m| - wpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['node'] + "-#{m}") - - 1.step(k8s['resources']['node']['count']) do |n| - next if m == n - system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{n}") - end - - 1.step(k8s['resources']['master']['count']) do |e| - system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{e}") - end - end - end - end end diff --git a/kubernetes/ha/config.yaml b/kubernetes/ha/config.yaml index bec3196..3347800 100644 --- a/kubernetes/ha/config.yaml +++ b/kubernetes/ha/config.yaml @@ -7,6 +7,7 @@ user: "vagrant" cluster: master: "controller" node: "worker" + ha: "load-balancer" resources: master: @@ -19,6 +20,10 @@ resources: memory: 2048 count: 2 ip_prefix: 20 + ha: + cpus: 1 + memory: 1024 + ip_prefix: 10 net: network_type: private_network diff --git a/kubernetes/ha/lib/ha.rb b/kubernetes/ha/lib/ha.rb new file mode 100644 index 0000000..05d3c08 --- /dev/null +++ b/kubernetes/ha/lib/ha.rb @@ -0,0 +1,49 @@ +config.vm.define "#{k8s['cluster']['ha']}" do |subconfig| + subconfig.vm.post_up_message = $msg + subconfig.vm.box = k8s['image'] + subconfig.vm.box_check_update = false + + subconfig.vm.hostname = "#{k8s['cluster']['ha']}" + subconfig.vm.network :private_network, ip: "#{k8s['ip_part']}.10" + + # Hostfile :: Master node + subconfig.vm.provision "Load Balancer hostfile update", type: "shell" do |lb| + lb.inline = <<-SHELL + echo -e "127.0.0.1\t$2" | tee -a /etc/hosts; echo -e "$1\t$2" | tee -a /etc/hosts + SHELL + lb.args = ["#{k8s['ip_part']}.10", "#{k8s['cluster']['ha']}"] + end + subconfig.vm.provision "Master and Worker node hostfile update", type: "shell" do |cluster| + cluster.inline = <<-SHELL + # master + for i in $(eval echo {1..#{k8s['resources']['master']['count']}}); do + echo -e "${1}.$((10 + $i))\t#{k8s['cluster']['master']}-${i}" | tee -a /etc/hosts + done + # worker + for i in $(eval echo {1..#{k8s['resources']['node']['count']}}); do + echo -e "${1}.$((20 + $i))\t#{k8s['cluster']['node']}-${i}" | tee -a /etc/hosts + done + SHELL + cluster.args = ["#{k8s['ip_part']}"] + end + + subconfig.vm.provider "virtualbox" do |vb| + vb.memory = k8s['resources']['ha']['memory'] + vb.cpus = k8s['resources']['ha']['cpus'] + vb.name = "#{k8s['cluster']['ha']}" + vb.gui = false + end + + subconfig.vm.provision "#{k8s['cluster']['master']}-setup", type: "shell" do |lb| + lb.path = "script/bootstrap_ha.sh" + lb.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "#{k8s['cluster']['master']}", "#{k8s['resources']['master']['count']}"] + end + + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| + reboot.privileged = true + reboot.inline = <<-SHELL + echo "----------------------------------|| Reboot to load all config" + SHELL + reboot.reboot = true + end +end diff --git a/kubernetes/ha/lib/master.rb b/kubernetes/ha/lib/master.rb index 18c36ee..42ba27c 100644 --- a/kubernetes/ha/lib/master.rb +++ b/kubernetes/ha/lib/master.rb @@ -9,9 +9,9 @@ # Hostfile :: Master node subconfig.vm.provision "Load Balancer hostfile update", type: "shell" do |lb| lb.inline = <<-SHELL - echo -e "127.0.0.1\t$1" | tee -a /etc/hosts + echo -e "127.0.0.1\t$1" | tee -a /etc/hosts; echo -e "$2\t$3" | tee -a /etc/hosts SHELL - lb.args = ["#{k8s['cluster']['master']}-#{i}"] + lb.args = ["#{k8s['cluster']['master']}-#{i}", "#{k8s['ip_part']}.#{k8s['resources']['ha']['ip_prefix']}", "#{k8s['cluster']['ha']}"] end subconfig.vm.provision "Master and Worker node hostfile update", type: "shell" do |cluster| cluster.inline = <<-SHELL @@ -35,6 +35,11 @@ vb.gui = false end + subconfig.vm.provision "vm-setup", type: "shell" do |vms| + vms.path = "script/bootstrap.sh" + vms.args = ["#{k8s['user']}"] + end + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| reboot.privileged = true reboot.inline = <<-SHELL diff --git a/kubernetes/ha/lib/node.rb b/kubernetes/ha/lib/node.rb index c7edd4f..75c1d19 100644 --- a/kubernetes/ha/lib/node.rb +++ b/kubernetes/ha/lib/node.rb @@ -7,9 +7,9 @@ # Hostfile :: Master node subconfig.vm.provision "Load Balancer hostfile update", type: "shell" do |lb| lb.inline = <<-SHELL - echo -e "127.0.0.1\t$1" | tee -a /etc/hosts + echo -e "127.0.0.1\t$1" | tee -a /etc/hosts; echo -e "$2\t$3" | tee -a /etc/hosts SHELL - lb.args = ["#{k8s['cluster']['node']}"] + lb.args = ["#{k8s['cluster']['node']}", "#{k8s['ip_part']}.#{k8s['resources']['ha']['ip_prefix']}", "#{k8s['cluster']['ha']}"] end subconfig.vm.provision "Master and Worker node hostfile update", type: "shell" do |cluster| cluster.inline = <<-SHELL @@ -33,7 +33,12 @@ vb.gui = false end - subconfig.vm.provision "firewall update", type: "shell" do |s| + subconfig.vm.provision "vm-setup", type: "shell" do |vms| + vms.path = "script/bootstrap.sh" + vms.args = ["#{k8s['user']}"] + end + + subconfig.vm.provision "firewall updates", type: "shell" do |s| s.inline = <<-SHELL ufw allow 10250/tcp ufw allow 10251/tcp diff --git a/kubernetes/ha/lib/ssh.rb b/kubernetes/ha/lib/ssh.rb new file mode 100644 index 0000000..b596640 --- /dev/null +++ b/kubernetes/ha/lib/ssh.rb @@ -0,0 +1,40 @@ +config.trigger.after :up do |trigger| + trigger.only_on = "#{k8s['cluster']['node']}-#{k8s['resources']['node']['count']}" + trigger.info = msg + + trigger.ruby do |env,machine| + lbpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['ha']) + + 1.step(k8s['resources']['master']['count']) do |m| + mpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['master'] + "-#{m}") + system("vagrant ssh --no-tty -c 'echo \"#{lbpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{m}") + + 1.step(k8s['resources']['master']['count']) do |n| + next if m == n + system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{n}") + end + + 1.step(k8s['resources']['node']['count']) do |e| + system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{e}") + end + + system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['ha']) + end + + 1.step(k8s['resources']['node']['count']) do |m| + wpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['node'] + "-#{m}") + system("vagrant ssh --no-tty -c 'echo \"#{lbpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{m}") + + 1.step(k8s['resources']['node']['count']) do |n| + next if m == n + system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{n}") + end + + 1.step(k8s['resources']['master']['count']) do |e| + system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{e}") + end + + system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['ha']) + end + end +end \ No newline at end of file diff --git a/kubernetes/ha/script/bootstrap_ha.sh b/kubernetes/ha/script/bootstrap_ha.sh new file mode 100644 index 0000000..4739ff9 --- /dev/null +++ b/kubernetes/ha/script/bootstrap_ha.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +apt-get update +add-apt-repository ppa:vbernat/haproxy-1.8 --yes +apt-get update +apt-get install -qq -y haproxy curl wget zip unzip telnet + +wget -q https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_linux_amd64 -O /usr/local/bin/cfssl +wget -q https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_linux_amd64 -O /usr/local/bin/cfssljson +chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson + +sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab +swapoff -a + +tee -a /etc/haproxy/haproxy.cfg << END + frontend kubernetes + bind 10.10.10.93:6443 + option tcplog + mode tcp + default_backend kubernetes-master-nodes +backend kubernetes-master-nodes + mode tcp + balance roundrobin + option tcp-check +$(for i in $(eval echo {1..$4}); do + echo " server $3-$i $2.$((10 + $i)):6443 check fall 3 rise 2" +done) +END +systemctl enable --now haproxy + +modprobe overlay +modprobe br_netfilter + +echo "ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa << Date: Sat, 1 May 2021 16:06:49 +0530 Subject: [PATCH 08/20] ha: Provisioning Certificates --- kubernetes/ha/lib/ha.rb | 7 +- kubernetes/ha/script/bootstrap_ha.sh | 6 +- kubernetes/ha/script/provisioning.sh | 278 +++++++++++++++++++++++++++ 3 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 kubernetes/ha/script/provisioning.sh diff --git a/kubernetes/ha/lib/ha.rb b/kubernetes/ha/lib/ha.rb index 05d3c08..f82e7d0 100644 --- a/kubernetes/ha/lib/ha.rb +++ b/kubernetes/ha/lib/ha.rb @@ -34,11 +34,16 @@ vb.gui = false end - subconfig.vm.provision "#{k8s['cluster']['master']}-setup", type: "shell" do |lb| + subconfig.vm.provision "#{k8s['cluster']['ha']}-setup", type: "shell" do |lb| lb.path = "script/bootstrap_ha.sh" lb.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "#{k8s['cluster']['master']}", "#{k8s['resources']['master']['count']}"] end + subconfig.vm.provision "#{k8s['cluster']['ha']}-setup", type: "shell" do |lb| + lb.path = "script/provisioning.sh" + lb.args = ["#{k8s['ip_part']}", "#{k8s['resources']['master']['ip_prefix']}", "#{k8s['resources']['node']['ip_prefix']}", "#{k8s['resources']['ha']['ip_prefix']}", "#{k8s['cluster']['master']}", "#{k8s['cluster']['node']}", "#{k8s['resources']['master']['count']}", "#{k8s['resources']['node']['count']}"] + end + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| reboot.privileged = true reboot.inline = <<-SHELL diff --git a/kubernetes/ha/script/bootstrap_ha.sh b/kubernetes/ha/script/bootstrap_ha.sh index 4739ff9..f8642fd 100644 --- a/kubernetes/ha/script/bootstrap_ha.sh +++ b/kubernetes/ha/script/bootstrap_ha.sh @@ -5,9 +5,9 @@ add-apt-repository ppa:vbernat/haproxy-1.8 --yes apt-get update apt-get install -qq -y haproxy curl wget zip unzip telnet -wget -q https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_linux_amd64 -O /usr/local/bin/cfssl -wget -q https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_linux_amd64 -O /usr/local/bin/cfssljson -chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson +# wget -q https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_linux_amd64 -O /usr/local/bin/cfssl +# wget -q https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_linux_amd64 -O /usr/local/bin/cfssljson +# chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab swapoff -a diff --git a/kubernetes/ha/script/provisioning.sh b/kubernetes/ha/script/provisioning.sh new file mode 100644 index 0000000..eb64832 --- /dev/null +++ b/kubernetes/ha/script/provisioning.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +wget -q --https-only https://storage.googleapis.com/kubernetes-the-hard-way/cfssl/1.4.1/linux/cfssl https://storage.googleapis.com/kubernetes-the-hard-way/cfssl/1.4.1/linux/cfssljson +chmod +x cfssl cfssljson +mv cfssl cfssljson /usr/local/bin/ + +wget -q --https-only https://storage.googleapis.com/kubernetes-release/release/v1.18.6/bin/linux/amd64/kubectl +chmod +x kubectl +mv kubectl /usr/local/bin/ + +mkdir -p /opt/certificates && chown vagrant -R $_ && cd $_ + +# Certificate Authority +{ +cat > ca-config.json < ca-csr.json < admin-csr.json < ${instance}-csr.json < kube-controller-manager-csr.json < kube-proxy-csr.json < kube-scheduler-csr.json < kubernetes-csr.json < service-account-csr.json < Date: Sat, 1 May 2021 17:31:43 +0530 Subject: [PATCH 09/20] ha: trigger update --- kubernetes/ha/Vagrantfile | 4 ++-- kubernetes/ha/lib/{ssh.rb => trigger.rb} | 0 kubernetes/ha/script/provisioning.sh | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) rename kubernetes/ha/lib/{ssh.rb => trigger.rb} (100%) diff --git a/kubernetes/ha/Vagrantfile b/kubernetes/ha/Vagrantfile index e2175ce..932141d 100644 --- a/kubernetes/ha/Vagrantfile +++ b/kubernetes/ha/Vagrantfile @@ -41,7 +41,7 @@ Vagrant.configure(k8s['api_version']) do |config| end # Exchange ssh keys to access each other, expect HA can access each of vm but not other vm cannot to access HA directly. - if File.exist?('lib/ssh.rb') - eval(IO.read('lib/ssh.rb'), binding) + if File.exist?('lib/trigger.rb') + eval(IO.read('lib/trigger.rb'), binding) end end diff --git a/kubernetes/ha/lib/ssh.rb b/kubernetes/ha/lib/trigger.rb similarity index 100% rename from kubernetes/ha/lib/ssh.rb rename to kubernetes/ha/lib/trigger.rb diff --git a/kubernetes/ha/script/provisioning.sh b/kubernetes/ha/script/provisioning.sh index eb64832..76e3d9c 100644 --- a/kubernetes/ha/script/provisioning.sh +++ b/kubernetes/ha/script/provisioning.sh @@ -215,7 +215,6 @@ for i in $(eval echo {1..$7}); do ips="${ips}$1.$(($2 + $i))," done instance="${ips}${instance}" -echo "-----------------------------------//${instance}" cat > kubernetes-csr.json < Date: Sat, 1 May 2021 18:41:52 +0530 Subject: [PATCH 10/20] ha: Distribute Certificates --- kubernetes/ha/lib/ha.rb | 6 +++--- kubernetes/ha/lib/trigger.rb | 8 +++++--- kubernetes/ha/script/provisioning.sh | 2 ++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kubernetes/ha/lib/ha.rb b/kubernetes/ha/lib/ha.rb index f82e7d0..cdec039 100644 --- a/kubernetes/ha/lib/ha.rb +++ b/kubernetes/ha/lib/ha.rb @@ -39,9 +39,9 @@ lb.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "#{k8s['cluster']['master']}", "#{k8s['resources']['master']['count']}"] end - subconfig.vm.provision "#{k8s['cluster']['ha']}-setup", type: "shell" do |lb| - lb.path = "script/provisioning.sh" - lb.args = ["#{k8s['ip_part']}", "#{k8s['resources']['master']['ip_prefix']}", "#{k8s['resources']['node']['ip_prefix']}", "#{k8s['resources']['ha']['ip_prefix']}", "#{k8s['cluster']['master']}", "#{k8s['cluster']['node']}", "#{k8s['resources']['master']['count']}", "#{k8s['resources']['node']['count']}"] + subconfig.vm.provision "certificates provisioning", type: "shell" do |lb_cert| + lb_cert.path = "script/provisioning.sh" + lb_cert.args = ["#{k8s['ip_part']}", "#{k8s['resources']['master']['ip_prefix']}", "#{k8s['resources']['node']['ip_prefix']}", "#{k8s['resources']['ha']['ip_prefix']}", "#{k8s['cluster']['master']}", "#{k8s['cluster']['node']}", "#{k8s['resources']['master']['count']}", "#{k8s['resources']['node']['count']}"] end subconfig.vm.provision "Restart VM", type: "shell" do |reboot| diff --git a/kubernetes/ha/lib/trigger.rb b/kubernetes/ha/lib/trigger.rb index b596640..1acf5be 100644 --- a/kubernetes/ha/lib/trigger.rb +++ b/kubernetes/ha/lib/trigger.rb @@ -8,6 +8,7 @@ 1.step(k8s['resources']['master']['count']) do |m| mpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['master'] + "-#{m}") system("vagrant ssh --no-tty -c 'echo \"#{lbpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{m}") + system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['ha']) 1.step(k8s['resources']['master']['count']) do |n| next if m == n @@ -18,12 +19,13 @@ system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{e}") end - system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['ha']) + system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/ca.pem /opt/certificates/ca-key.pem /opt/certificates/kubernetes-key.pem /opt/certificates/kubernetes.pem /opt/certificates/service-account-key.pem /opt/certificates/service-account.pem " + k8s['cluster']['master'] + "-#{m}" + ":~/' " + k8s['cluster']['ha']) end 1.step(k8s['resources']['node']['count']) do |m| wpub, stdeerr, status = Open3.capture3("vagrant ssh --no-tty -c 'cat /home/" + k8s['user'] + "/.ssh/id_rsa.pub' " + k8s['cluster']['node'] + "-#{m}") system("vagrant ssh --no-tty -c 'echo \"#{lbpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{m}") + system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['ha']) 1.step(k8s['resources']['node']['count']) do |n| next if m == n @@ -34,7 +36,7 @@ system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{e}") end - system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['ha']) + system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/ca.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}-key.pem " + k8s['cluster']['node'] + "-#{m}" + ":~/' " + k8s['cluster']['ha']) end end -end \ No newline at end of file +end diff --git a/kubernetes/ha/script/provisioning.sh b/kubernetes/ha/script/provisioning.sh index 76e3d9c..498e30e 100644 --- a/kubernetes/ha/script/provisioning.sh +++ b/kubernetes/ha/script/provisioning.sh @@ -275,3 +275,5 @@ cfssl gencert \ service-account-csr.json | cfssljson -bare service-account } + +chown vagrant -R /opt/certificates From 97995c791a6c8ac6b80499928e5141661586101d Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Sat, 1 May 2021 20:34:00 +0530 Subject: [PATCH 11/20] ha: etcd setup --- kubernetes/ha/lib/ha.rb | 5 ++ kubernetes/ha/lib/master.rb | 10 ++-- kubernetes/ha/lib/trigger.rb | 8 ++- kubernetes/ha/script/bootstrap.sh | 1 + kubernetes/ha/script/bootstrap_ha.sh | 4 -- kubernetes/ha/script/bootstrap_master.sh | 58 +++++++++++++++++--- kubernetes/ha/script/kube_config.sh | 67 ++++++++++++++++++++++++ kubernetes/ha/script/provisioning.sh | 2 +- 8 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 kubernetes/ha/script/kube_config.sh diff --git a/kubernetes/ha/lib/ha.rb b/kubernetes/ha/lib/ha.rb index cdec039..4c03c5f 100644 --- a/kubernetes/ha/lib/ha.rb +++ b/kubernetes/ha/lib/ha.rb @@ -44,6 +44,11 @@ lb_cert.args = ["#{k8s['ip_part']}", "#{k8s['resources']['master']['ip_prefix']}", "#{k8s['resources']['node']['ip_prefix']}", "#{k8s['resources']['ha']['ip_prefix']}", "#{k8s['cluster']['master']}", "#{k8s['cluster']['node']}", "#{k8s['resources']['master']['count']}", "#{k8s['resources']['node']['count']}"] end + subconfig.vm.provision "Generating Kubernetes Configuration", type: "shell" do |lb_config| + lb_config.path = "script/kube_config.sh" + lb_config.args = ["#{k8s['ip_part']}", "#{k8s['resources']['master']['ip_prefix']}", "#{k8s['resources']['node']['ip_prefix']}", "#{k8s['resources']['ha']['ip_prefix']}", "#{k8s['cluster']['master']}", "#{k8s['cluster']['node']}", "#{k8s['resources']['master']['count']}", "#{k8s['resources']['node']['count']}"] + end + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| reboot.privileged = true reboot.inline = <<-SHELL diff --git a/kubernetes/ha/lib/master.rb b/kubernetes/ha/lib/master.rb index 42ba27c..e3cec23 100644 --- a/kubernetes/ha/lib/master.rb +++ b/kubernetes/ha/lib/master.rb @@ -40,6 +40,11 @@ vms.args = ["#{k8s['user']}"] end + subconfig.vm.provision "#{k8s['cluster']['master']}-#{i}-setup", type: "shell" do |mns| + mns.path = "script/bootstrap_master.sh" + mns.args = ["#{k8s['ip_part']}", "#{k8s['resources']['master']['ip_prefix']}", "#{i}", "#{k8s['cluster']['master']}", "#{k8s['resources']['master']['count']}"] + end + subconfig.vm.provision "Restart VM", type: "shell" do |reboot| reboot.privileged = true reboot.inline = <<-SHELL @@ -47,9 +52,4 @@ SHELL reboot.reboot = true end - - subconfig.vm.provision "#{k8s['cluster']['master']}-setup", type: "shell" do |mns| - mns.path = "script/bootstrap_master.sh" - mns.args = ["#{k8s['user']}", "#{k8s['ip_part']}", "10"] - end end diff --git a/kubernetes/ha/lib/trigger.rb b/kubernetes/ha/lib/trigger.rb index 1acf5be..287c95d 100644 --- a/kubernetes/ha/lib/trigger.rb +++ b/kubernetes/ha/lib/trigger.rb @@ -19,7 +19,10 @@ system("vagrant ssh --no-tty -c 'echo \"#{mpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['node'] + "-#{e}") end - system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/ca.pem /opt/certificates/ca-key.pem /opt/certificates/kubernetes-key.pem /opt/certificates/kubernetes.pem /opt/certificates/service-account-key.pem /opt/certificates/service-account.pem " + k8s['cluster']['master'] + "-#{m}" + ":~/' " + k8s['cluster']['ha']) + # Push all required configs/certificates to master node + system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/encryption-config.yaml /opt/certificates/kube-controller-manager.kubeconfig /opt/certificates/kube-scheduler.kubeconfig /opt/certificates/admin.kubeconfig /opt/certificates/ca.pem /opt/certificates/ca-key.pem /opt/certificates/kubernetes-key.pem /opt/certificates/kubernetes.pem /opt/certificates/service-account-key.pem /opt/certificates/service-account.pem " + k8s['cluster']['master'] + "-#{m}" + ":~/certificates/' " + k8s['cluster']['ha']) + # Start etcd on all controller + system("vagrant ssh --no-tty -c 'cp /home/vagrant/certificates/ca.pem /home/vagrant/certificates/kubernetes-key.pem /home/vagrant/certificates/kubernetes.pem /etc/etcd/; systemctl enable --now etcd' " + k8s['cluster']['master'] + "-#{m}") end 1.step(k8s['resources']['node']['count']) do |m| @@ -36,7 +39,8 @@ system("vagrant ssh --no-tty -c 'echo \"#{wpub}\" >> /home/" + k8s['user'] + "/.ssh/authorized_keys' " + k8s['cluster']['master'] + "-#{e}") end - system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/ca.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}-key.pem " + k8s['cluster']['node'] + "-#{m}" + ":~/' " + k8s['cluster']['ha']) + # Push all required configs/certificates to worker node + system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/" + k8s['cluster']['node'] + "-#{m}.kubeconfig" + " /opt/certificates/kube-proxy.kubeconfig /opt/certificates/ca.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}-key.pem " + k8s['cluster']['node'] + "-#{m}" + ":~/certificates/' " + k8s['cluster']['ha']) end end end diff --git a/kubernetes/ha/script/bootstrap.sh b/kubernetes/ha/script/bootstrap.sh index 43e6029..95d8676 100644 --- a/kubernetes/ha/script/bootstrap.sh +++ b/kubernetes/ha/script/bootstrap.sh @@ -48,6 +48,7 @@ sed -i "s/DEFAULT_FORWARD_POLICY=\"DROP\"/DEFAULT_FORWARD_POLICY=\"ACCEPT\"/g" / sed -i '/net\/ipv4\/ip_forward/s/^#//g' /etc/ufw/sysctl.conf sed -i '/net\/ipv4\/conf\/all\/forwarding/s/^#//g' /etc/ufw/sysctl.conf sed -i '/net\/ipv6\/conf\/default\/forwarding/s/^#//g' /etc/ufw/sysctl.conf +mkdir -p /home/vagrant/certificates && chown vagrant:vagrant -R $_ ufw enable <<> $HOME/.bash_profile' -# chown ${1} /etc/kubernetes/admin.conf -# echo "export KUBEADM_JOIN=\"${join_command}\"" >> /home/${1}/.bash_profile + mkdir -p /etc/etcd /var/lib/etcd + chmod 700 /var/lib/etcd +} -# su ${1} -c "kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml" +INTERNAL_IP="${1}.$(($2 + $3))" +ETCD_NAME="${4}-${3}" +INSTANCE="" + +for i in $(eval echo {1..$5}); do + INSTANCE="${INSTANCE}$4-$i=https://$1.$(($2 + $i)):2380," +done + +cat < encryption-config.yaml < ${instance}-csr.json < Date: Sat, 1 May 2021 22:25:05 +0530 Subject: [PATCH 12/20] ha: master setup --- kubernetes/ha/lib/trigger.rb | 4 +- kubernetes/ha/script/bootstrap_master.sh | 138 +++++++++++++++++++++++ 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/kubernetes/ha/lib/trigger.rb b/kubernetes/ha/lib/trigger.rb index 287c95d..9268926 100644 --- a/kubernetes/ha/lib/trigger.rb +++ b/kubernetes/ha/lib/trigger.rb @@ -20,9 +20,9 @@ end # Push all required configs/certificates to master node - system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/encryption-config.yaml /opt/certificates/kube-controller-manager.kubeconfig /opt/certificates/kube-scheduler.kubeconfig /opt/certificates/admin.kubeconfig /opt/certificates/ca.pem /opt/certificates/ca-key.pem /opt/certificates/kubernetes-key.pem /opt/certificates/kubernetes.pem /opt/certificates/service-account-key.pem /opt/certificates/service-account.pem " + k8s['cluster']['master'] + "-#{m}" + ":~/certificates/' " + k8s['cluster']['ha']) + system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/{encryption-config.yaml,kube-controller-manager.kubeconfig,kube-scheduler.kubeconfig,admin.kubeconfig,ca.pem,ca-key.pem,kubernetes-key.pem,kubernetes.pem,service-account-key.pem,service-account.pem} " + k8s['cluster']['master'] + "-#{m}" + ":~/certificates/' " + k8s['cluster']['ha']) # Start etcd on all controller - system("vagrant ssh --no-tty -c 'cp /home/vagrant/certificates/ca.pem /home/vagrant/certificates/kubernetes-key.pem /home/vagrant/certificates/kubernetes.pem /etc/etcd/; systemctl enable --now etcd' " + k8s['cluster']['master'] + "-#{m}") + system("vagrant ssh --no-tty -c 'sudo cp /home/vagrant/certificates/{ca.pem,kubernetes-key.pem,kubernetes.pem} /etc/etcd/; sudo cp /home/vagrant/certificates/{ca.pem,ca-key.pem,kubernetes-key.pem,kubernetes.pem,service-account-key.pem,service-account.pem,encryption-config.yaml} /var/lib/kubernetes/; sudo cp /home/vagrant/certificates/{kube-controller-manager.kubeconfig,kube-scheduler.kubeconfig} /var/lib/kubernetes/; sudo systemctl enable --now etcd; sudo systemctl enable --now kube-apiserver; sudo systemctl enable --now kube-controller-manager; sudo systemctl enable --now kube-scheduler; sudo systemctl enable --now nginx' " + k8s['cluster']['master'] + "-#{m}") end 1.step(k8s['resources']['node']['count']) do |m| diff --git a/kubernetes/ha/script/bootstrap_master.sh b/kubernetes/ha/script/bootstrap_master.sh index f0faa78..5928816 100644 --- a/kubernetes/ha/script/bootstrap_master.sh +++ b/kubernetes/ha/script/bootstrap_master.sh @@ -1,5 +1,20 @@ #!/usr/bin/env bash +apt-get update +apt-get install -y nginx + +cat > kubernetes.default.svc.cluster.local < Date: Sun, 2 May 2021 00:16:08 +0530 Subject: [PATCH 13/20] ha: worker setup --- kubernetes/ha/lib/node.rb | 10 +- kubernetes/ha/lib/trigger.rb | 4 + kubernetes/ha/script/bootstrap.sh | 36 ++--- kubernetes/ha/script/bootstrap_ha.sh | 2 +- kubernetes/ha/script/bootstrap_master.sh | 39 +++++ kubernetes/ha/script/bootstrap_node.sh | 175 +++++++++++++++++++++++ 6 files changed, 239 insertions(+), 27 deletions(-) create mode 100644 kubernetes/ha/script/bootstrap_node.sh diff --git a/kubernetes/ha/lib/node.rb b/kubernetes/ha/lib/node.rb index 75c1d19..86b7fda 100644 --- a/kubernetes/ha/lib/node.rb +++ b/kubernetes/ha/lib/node.rb @@ -38,14 +38,8 @@ vms.args = ["#{k8s['user']}"] end - subconfig.vm.provision "firewall updates", type: "shell" do |s| - s.inline = <<-SHELL - ufw allow 10250/tcp - ufw allow 10251/tcp - ufw allow 10255/tcp - ufw allow 30000:32767/tcp - ufw reload - SHELL + subconfig.vm.provision "kube-setup", type: "shell" do |ks| + ks.path = "script/bootstrap_node.sh" end subconfig.vm.provision "Restart VM", type: "shell" do |reboot| diff --git a/kubernetes/ha/lib/trigger.rb b/kubernetes/ha/lib/trigger.rb index 9268926..c15adbd 100644 --- a/kubernetes/ha/lib/trigger.rb +++ b/kubernetes/ha/lib/trigger.rb @@ -41,6 +41,10 @@ # Push all required configs/certificates to worker node system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/" + k8s['cluster']['node'] + "-#{m}.kubeconfig" + " /opt/certificates/kube-proxy.kubeconfig /opt/certificates/ca.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}-key.pem " + k8s['cluster']['node'] + "-#{m}" + ":~/certificates/' " + k8s['cluster']['ha']) + # Bootstrapping the Kubernetes Worker Nodes + system("vagrant ssh --no-tty -c 'sudo cp /home/vagrant/certificates/{" + k8s['cluster']['node'] + "-#{m}-key.pem," + k8s['cluster']['node'] + "-#{m}.pem} /var/lib/kubelet/; sudo cp /home/vagrant/certificates/" + k8s['cluster']['node'] + "-#{m}.kubeconfig /var/lib/kubelet/kubeconfig; sudo cp /home/vagrant/certificates/ca.pem /var/lib/kubernetes/; sudo cp /home/vagrant/certificates/kube-proxy.kubeconfig /var/lib/kube-proxy/kubeconfig; sudo systemctl enable --now kubelet; sudo systemctl enable --now kube-proxy; sudo systemctl enable --now containerd' " + k8s['cluster']['node'] + "-#{m}") end + + system("vagrant ssh --no-tty -c 'kubectl apply --kubeconfig /home/vagrant/certificates/admin.kubeconfig -f /home/vagrant/certificates/cluster_role.yaml; kubectl apply --kubeconfig /home/vagrant/certificates/admin.kubeconfig -f /home/vagrant/certificates/cluster_role_binding.yaml' " + k8s['cluster']['master'] + "-1") end end diff --git a/kubernetes/ha/script/bootstrap.sh b/kubernetes/ha/script/bootstrap.sh index 95d8676..c4e0f7e 100644 --- a/kubernetes/ha/script/bootstrap.sh +++ b/kubernetes/ha/script/bootstrap.sh @@ -18,24 +18,24 @@ swapoff -a apt-get update apt-get install -y apt-transport-https ca-certificates curl wget zip unzip vim git gnupg lsb-release software-properties-common telnet curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -apt-get update -apt-get install -y docker-ce docker-ce-cli containerd.io -usermod -aG docker ${1} - -cat < Date: Sun, 2 May 2021 00:24:21 +0530 Subject: [PATCH 14/20] ha: haproxy fix --- kubernetes/ha/script/bootstrap_ha.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kubernetes/ha/script/bootstrap_ha.sh b/kubernetes/ha/script/bootstrap_ha.sh index 9f7afdb..daeeda8 100644 --- a/kubernetes/ha/script/bootstrap_ha.sh +++ b/kubernetes/ha/script/bootstrap_ha.sh @@ -36,4 +36,8 @@ sed -i '/net\/ipv4\/conf\/all\/forwarding/s/^#//g' /etc/ufw/sysctl.conf sed -i '/net\/ipv6\/conf\/default\/forwarding/s/^#//g' /etc/ufw/sysctl.conf ufw enable << Date: Sun, 2 May 2021 00:24:59 +0530 Subject: [PATCH 15/20] ha: haproxy fix --- kubernetes/ha/script/bootstrap_ha.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/kubernetes/ha/script/bootstrap_ha.sh b/kubernetes/ha/script/bootstrap_ha.sh index daeeda8..ddf6f34 100644 --- a/kubernetes/ha/script/bootstrap_ha.sh +++ b/kubernetes/ha/script/bootstrap_ha.sh @@ -36,7 +36,6 @@ sed -i '/net\/ipv4\/conf\/all\/forwarding/s/^#//g' /etc/ufw/sysctl.conf sed -i '/net\/ipv6\/conf\/default\/forwarding/s/^#//g' /etc/ufw/sysctl.conf ufw enable << Date: Sun, 2 May 2021 09:26:47 +0530 Subject: [PATCH 16/20] ha: haproxy fix for start --- kubernetes/Bug_Control.md | 50 ++++++++++++++++++++++++++++ kubernetes/ha/Vagrantfile | 1 + kubernetes/ha/config.yaml | 4 +-- kubernetes/ha/script/bootstrap_ha.sh | 1 - 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 kubernetes/Bug_Control.md diff --git a/kubernetes/Bug_Control.md b/kubernetes/Bug_Control.md new file mode 100644 index 0000000..5a1cb75 --- /dev/null +++ b/kubernetes/Bug_Control.md @@ -0,0 +1,50 @@ +# Error Control Vagrant Kubernetes +While using `vagrant / vagrantfile` you can encounter of some errors, try to add few errors which faced during development of `kubernetes vagrant` file with solutions which worked whenever faced issues. + +### SSH connection get timeout or say ssh connection got reset, something like below error + +```bash +SSH connection was reset! This usually happens when the machine is +taking too long to reboot. First, try reloading your machine with +`vagrant reload`, since a simple restart sometimes fixes things. +If that doesn't work, destroy your machine and recreate it with +a `vagrant destroy` followed by a `vagrant up`. If that doesn't work, +contact support. +``` + +> Then can try to add below config to your vagrantfile +```bash + config.vm.boot_timeout = 600 +``` + +> if still not solved then best can try +```bash +$ rm -rf .vagrant +``` + +### While creating VM, vagrant failed to rename vm because of unclear vms +```bash +The name of your virtual machine couldn't be set because VirtualBox +is reporting another VM with that name already exists. Most of the +time, this is because of an error with VirtualBox not cleaning up +properly. To fix this, verify that no VMs with that name do exist +(by opening the VirtualBox GUI). If they don't, then look at the +folder in the error message from VirtualBox below and remove it +if there isn't any information you need in there. + +VirtualBox error: + +VBoxManage: error: Could not rename the directory '/Users/XXXXXXX/VirtualBox VMs/ubuntu-18.04-amd64_1619926105557_22107' to '/Users/XXXXXXX/VirtualBox VMs/load-balancer' to save the settings file (VERR_ALREADY_EXISTS) +VBoxManage: error: Details: code NS_ERROR_FAILURE (0x80004005), component SessionMachine, interface IMachine, callee nsISupports +VBoxManage: error: Context: "SaveSettings()" at line 3249 of file VBoxManageModifyVM.cpp +``` + +> Simple can run command with path from error +```bash +$ rm -rf /Users/XXXXXXX/VirtualBox VMs/load-balancer +``` + +> Better to comment below line from your vagrantfile +```bash + config.ssh.keep_alive = true +``` \ No newline at end of file diff --git a/kubernetes/ha/Vagrantfile b/kubernetes/ha/Vagrantfile index 932141d..053a640 100644 --- a/kubernetes/ha/Vagrantfile +++ b/kubernetes/ha/Vagrantfile @@ -20,6 +20,7 @@ MSG Vagrant.configure(k8s['api_version']) do |config| config.vm.boot_timeout = 600 + # config.ssh.keep_alive = true # Load Balancer vm if File.exist?('lib/ha.rb') diff --git a/kubernetes/ha/config.yaml b/kubernetes/ha/config.yaml index 3347800..d6ab347 100644 --- a/kubernetes/ha/config.yaml +++ b/kubernetes/ha/config.yaml @@ -11,8 +11,8 @@ cluster: resources: master: - cpus: 2 - memory: 2048 + cpus: 1 + memory: 1024 count: 3 ip_prefix: 10 node: diff --git a/kubernetes/ha/script/bootstrap_ha.sh b/kubernetes/ha/script/bootstrap_ha.sh index ddf6f34..e861358 100644 --- a/kubernetes/ha/script/bootstrap_ha.sh +++ b/kubernetes/ha/script/bootstrap_ha.sh @@ -22,7 +22,6 @@ $(for i in $(eval echo {1..$4}); do echo " server $3-$i $2.$((10 + $i)):6443 check fall 3 rise 2" done) END -systemctl enable --now haproxy modprobe overlay modprobe br_netfilter From ce8c5c86a2b52bed795fac253d126eac1a3a3ef8 Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Sun, 2 May 2021 10:38:13 +0530 Subject: [PATCH 17/20] ha: local kube config generate --- kubernetes/ha/lib/trigger.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kubernetes/ha/lib/trigger.rb b/kubernetes/ha/lib/trigger.rb index c15adbd..8843926 100644 --- a/kubernetes/ha/lib/trigger.rb +++ b/kubernetes/ha/lib/trigger.rb @@ -22,7 +22,7 @@ # Push all required configs/certificates to master node system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/{encryption-config.yaml,kube-controller-manager.kubeconfig,kube-scheduler.kubeconfig,admin.kubeconfig,ca.pem,ca-key.pem,kubernetes-key.pem,kubernetes.pem,service-account-key.pem,service-account.pem} " + k8s['cluster']['master'] + "-#{m}" + ":~/certificates/' " + k8s['cluster']['ha']) # Start etcd on all controller - system("vagrant ssh --no-tty -c 'sudo cp /home/vagrant/certificates/{ca.pem,kubernetes-key.pem,kubernetes.pem} /etc/etcd/; sudo cp /home/vagrant/certificates/{ca.pem,ca-key.pem,kubernetes-key.pem,kubernetes.pem,service-account-key.pem,service-account.pem,encryption-config.yaml} /var/lib/kubernetes/; sudo cp /home/vagrant/certificates/{kube-controller-manager.kubeconfig,kube-scheduler.kubeconfig} /var/lib/kubernetes/; sudo systemctl enable --now etcd; sudo systemctl enable --now kube-apiserver; sudo systemctl enable --now kube-controller-manager; sudo systemctl enable --now kube-scheduler; sudo systemctl enable --now nginx' " + k8s['cluster']['master'] + "-#{m}") + system("vagrant ssh --no-tty -c 'sudo cp /home/vagrant/certificates/{ca.pem,kubernetes-key.pem,kubernetes.pem} /etc/etcd/; sudo cp /home/vagrant/certificates/{ca.pem,ca-key.pem,kubernetes-key.pem,kubernetes.pem,service-account-key.pem,service-account.pem,encryption-config.yaml} /var/lib/kubernetes/; sudo cp /home/vagrant/certificates/{kube-controller-manager.kubeconfig,kube-scheduler.kubeconfig} /var/lib/kubernetes/; sudo systemctl enable --now etcd; sudo systemctl enable --now kube-apiserver; sudo systemctl enable --now kube-controller-manager; sudo systemctl enable --now kube-scheduler; sudo systemctl enable --now nginx; mkdir -p /home/" + k8s['user'] + "/.kube; cp -i /home/" + k8s['user'] + "/certificates/admin.kubeconfig /home/" + k8s['user'] + "/.kube/config' " + k8s['cluster']['master'] + "-#{m}") end 1.step(k8s['resources']['node']['count']) do |m| @@ -40,11 +40,14 @@ end # Push all required configs/certificates to worker node - system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/" + k8s['cluster']['node'] + "-#{m}.kubeconfig" + " /opt/certificates/kube-proxy.kubeconfig /opt/certificates/ca.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}.pem /opt/certificates/" + k8s['cluster']['node'] + "-#{m}-key.pem " + k8s['cluster']['node'] + "-#{m}" + ":~/certificates/' " + k8s['cluster']['ha']) + system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no /opt/certificates/{" + k8s['cluster']['node'] + "-#{m}.kubeconfig" + ",kube-proxy.kubeconfig,ca.pem,admin.kubeconfig," + k8s['cluster']['node'] + "-#{m}.pem," + k8s['cluster']['node'] + "-#{m}-key.pem} " + k8s['cluster']['node'] + "-#{m}" + ":~/certificates/' " + k8s['cluster']['ha']) # Bootstrapping the Kubernetes Worker Nodes - system("vagrant ssh --no-tty -c 'sudo cp /home/vagrant/certificates/{" + k8s['cluster']['node'] + "-#{m}-key.pem," + k8s['cluster']['node'] + "-#{m}.pem} /var/lib/kubelet/; sudo cp /home/vagrant/certificates/" + k8s['cluster']['node'] + "-#{m}.kubeconfig /var/lib/kubelet/kubeconfig; sudo cp /home/vagrant/certificates/ca.pem /var/lib/kubernetes/; sudo cp /home/vagrant/certificates/kube-proxy.kubeconfig /var/lib/kube-proxy/kubeconfig; sudo systemctl enable --now kubelet; sudo systemctl enable --now kube-proxy; sudo systemctl enable --now containerd' " + k8s['cluster']['node'] + "-#{m}") + system("vagrant ssh --no-tty -c 'sudo cp /home/vagrant/certificates/{" + k8s['cluster']['node'] + "-#{m}-key.pem," + k8s['cluster']['node'] + "-#{m}.pem} /var/lib/kubelet/; sudo cp /home/vagrant/certificates/" + k8s['cluster']['node'] + "-#{m}.kubeconfig /var/lib/kubelet/kubeconfig; sudo cp /home/vagrant/certificates/ca.pem /var/lib/kubernetes/; sudo cp /home/vagrant/certificates/kube-proxy.kubeconfig /var/lib/kube-proxy/kubeconfig; sudo systemctl enable --now kubelet; sudo systemctl enable --now kube-proxy; sudo systemctl enable --now containerd; mkdir -p /home/" + k8s['user'] + "/.kube; cp -i /home/" + k8s['user'] + "/certificates/admin.kubeconfig /home/" + k8s['user'] + "/.kube/config' " + k8s['cluster']['node'] + "-#{m}") end system("vagrant ssh --no-tty -c 'kubectl apply --kubeconfig /home/vagrant/certificates/admin.kubeconfig -f /home/vagrant/certificates/cluster_role.yaml; kubectl apply --kubeconfig /home/vagrant/certificates/admin.kubeconfig -f /home/vagrant/certificates/cluster_role_binding.yaml' " + k8s['cluster']['master'] + "-1") + # copy pem files in local to generate kube config + system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no " + k8s['cluster']['ha'] + ":/opt/certificates/{ca.pem,admin.pem,admin-key.pem} /tmp/") + system("kubectl config set-cluster kubernetes-the-hard-way --certificate-authority=/tmp/ca.pem --embed-certs=true --server=https://" + k8s['ip_part'] + "." + k8s['resources']['ha']['ip_prefix'] + ":6443 && kubectl config set-credentials admin --client-certificate=/tmp/admin.pem --client-key=/tmp/admin-key.pem && kubectl config set-context kubernetes-the-hard-way --cluster=kubernetes-the-hard-way --user=admin && kubectl config use-context kubernetes-the-hard-way") end end From 87bd2b5e1b3dc2516a9dd387c9d5a691ff47e437 Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Sun, 2 May 2021 11:44:45 +0530 Subject: [PATCH 18/20] ha: disable man-db to fast boot --- kubernetes/ha/lib/trigger.rb | 10 +++++--- kubernetes/ha/script/bootstrap.sh | 38 +++++++++++++++------------- kubernetes/ha/script/bootstrap_ha.sh | 20 +++++++++++++++ 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/kubernetes/ha/lib/trigger.rb b/kubernetes/ha/lib/trigger.rb index 8843926..fb531a4 100644 --- a/kubernetes/ha/lib/trigger.rb +++ b/kubernetes/ha/lib/trigger.rb @@ -46,8 +46,12 @@ end system("vagrant ssh --no-tty -c 'kubectl apply --kubeconfig /home/vagrant/certificates/admin.kubeconfig -f /home/vagrant/certificates/cluster_role.yaml; kubectl apply --kubeconfig /home/vagrant/certificates/admin.kubeconfig -f /home/vagrant/certificates/cluster_role_binding.yaml' " + k8s['cluster']['master'] + "-1") - # copy pem files in local to generate kube config - system("vagrant ssh --no-tty -c 'scp -o StrictHostKeyChecking=no " + k8s['cluster']['ha'] + ":/opt/certificates/{ca.pem,admin.pem,admin-key.pem} /tmp/") - system("kubectl config set-cluster kubernetes-the-hard-way --certificate-authority=/tmp/ca.pem --embed-certs=true --server=https://" + k8s['ip_part'] + "." + k8s['resources']['ha']['ip_prefix'] + ":6443 && kubectl config set-credentials admin --client-certificate=/tmp/admin.pem --client-key=/tmp/admin-key.pem && kubectl config set-context kubernetes-the-hard-way --cluster=kubernetes-the-hard-way --user=admin && kubectl config use-context kubernetes-the-hard-way") + + # Configuring kubectl for Remote Access + system("mkdir -p ${HOME}/.kube") + system("vagrant ssh --no-tty -c 'cat /opt/certificates/ca.pem' " + k8s['cluster']['ha'] + " > ${HOME}/.kube/ca.pem") + system("vagrant ssh --no-tty -c 'cat /opt/certificates/admin.pem' " + k8s['cluster']['ha'] + " > ${HOME}/.kube/admin.pem") + system("vagrant ssh --no-tty -c 'cat /opt/certificates/admin-key.pem' " + k8s['cluster']['ha'] + " > ${HOME}/.kube/admin-key.pem") + system("kubectl config set-cluster kubernetes-the-hard-way --certificate-authority=${HOME}/.kube/ca.pem --embed-certs=true --server=https://#{k8s['ip_part']}.#{k8s['resources']['ha']['ip_prefix']}:6443 && kubectl config set-credentials admin --client-certificate=${HOME}/.kube/admin.pem --client-key=${HOME}/.kube/admin-key.pem && kubectl config set-context kubernetes-the-hard-way --cluster=kubernetes-the-hard-way --user=admin && kubectl config use-context kubernetes-the-hard-way") end end diff --git a/kubernetes/ha/script/bootstrap.sh b/kubernetes/ha/script/bootstrap.sh index c4e0f7e..973aacb 100644 --- a/kubernetes/ha/script/bootstrap.sh +++ b/kubernetes/ha/script/bootstrap.sh @@ -15,27 +15,29 @@ sysctl --system sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab swapoff -a +# disable man-db installation +{ +apt-get remove man-db --purge -y +sudo rm -rf /usr/share/locale/ +sudo rm -rf /usr/share/man/ +sudo rm -rf /usr/share/doc/ + +cat > /etc/dpkg/dpkg.cfg.d/01_nodoc < /etc/dpkg/dpkg.cfg.d/01_nodoc < Date: Sun, 2 May 2021 12:24:57 +0530 Subject: [PATCH 19/20] ha: ssh timeout --- kubernetes/ha/lib/trigger.rb | 3 +++ kubernetes/ha/script/bootstrap.sh | 4 ++++ kubernetes/ha/script/bootstrap_ha.sh | 6 +++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/kubernetes/ha/lib/trigger.rb b/kubernetes/ha/lib/trigger.rb index fb531a4..136bc0d 100644 --- a/kubernetes/ha/lib/trigger.rb +++ b/kubernetes/ha/lib/trigger.rb @@ -53,5 +53,8 @@ system("vagrant ssh --no-tty -c 'cat /opt/certificates/admin.pem' " + k8s['cluster']['ha'] + " > ${HOME}/.kube/admin.pem") system("vagrant ssh --no-tty -c 'cat /opt/certificates/admin-key.pem' " + k8s['cluster']['ha'] + " > ${HOME}/.kube/admin-key.pem") system("kubectl config set-cluster kubernetes-the-hard-way --certificate-authority=${HOME}/.kube/ca.pem --embed-certs=true --server=https://#{k8s['ip_part']}.#{k8s['resources']['ha']['ip_prefix']}:6443 && kubectl config set-credentials admin --client-certificate=${HOME}/.kube/admin.pem --client-key=${HOME}/.kube/admin-key.pem && kubectl config set-context kubernetes-the-hard-way --cluster=kubernetes-the-hard-way --user=admin && kubectl config use-context kubernetes-the-hard-way") + + # Deploying the DNS Cluster Add-on + system("kubectl apply -f https://storage.googleapis.com/kubernetes-the-hard-way/coredns-1.8.yaml") end end diff --git a/kubernetes/ha/script/bootstrap.sh b/kubernetes/ha/script/bootstrap.sh index 973aacb..91acaf9 100644 --- a/kubernetes/ha/script/bootstrap.sh +++ b/kubernetes/ha/script/bootstrap.sh @@ -54,3 +54,7 @@ mkdir -p /home/vagrant/certificates && chown vagrant:vagrant -R $_ ufw enable <<> /etc/ssh/ssh_config +echo "ClientAliveCountMax 5" >> /etc/ssh/ssh_config \ No newline at end of file diff --git a/kubernetes/ha/script/bootstrap_ha.sh b/kubernetes/ha/script/bootstrap_ha.sh index e115c82..07036ee 100644 --- a/kubernetes/ha/script/bootstrap_ha.sh +++ b/kubernetes/ha/script/bootstrap_ha.sh @@ -58,4 +58,8 @@ ufw enable <<> /etc/ssh/ssh_config +echo "ClientAliveCountMax 5" >> /etc/ssh/ssh_config \ No newline at end of file From ce0a0b2b8a84a6c37a6aa5ee59189d70722d6e94 Mon Sep 17 00:00:00 2001 From: Anand Kumar Martund Date: Sun, 2 May 2021 14:23:23 +0530 Subject: [PATCH 20/20] ha: fix for ssh timeout --- kubernetes/ha/lib/ha.rb | 8 +------- kubernetes/ha/lib/master.rb | 8 +------- kubernetes/ha/lib/node.rb | 8 +------- kubernetes/ha/script/bootstrap.sh | 3 +-- kubernetes/ha/script/bootstrap_ha.sh | 3 +-- 5 files changed, 5 insertions(+), 25 deletions(-) diff --git a/kubernetes/ha/lib/ha.rb b/kubernetes/ha/lib/ha.rb index 4c03c5f..049d812 100644 --- a/kubernetes/ha/lib/ha.rb +++ b/kubernetes/ha/lib/ha.rb @@ -49,11 +49,5 @@ lb_config.args = ["#{k8s['ip_part']}", "#{k8s['resources']['master']['ip_prefix']}", "#{k8s['resources']['node']['ip_prefix']}", "#{k8s['resources']['ha']['ip_prefix']}", "#{k8s['cluster']['master']}", "#{k8s['cluster']['node']}", "#{k8s['resources']['master']['count']}", "#{k8s['resources']['node']['count']}"] end - subconfig.vm.provision "Restart VM", type: "shell" do |reboot| - reboot.privileged = true - reboot.inline = <<-SHELL - echo "----------------------------------|| Reboot to load all config" - SHELL - reboot.reboot = true - end + subconfig.vm.provision "Reboot to load all config", type:"shell", inline: "shutdown -r now" end diff --git a/kubernetes/ha/lib/master.rb b/kubernetes/ha/lib/master.rb index e3cec23..5fb1261 100644 --- a/kubernetes/ha/lib/master.rb +++ b/kubernetes/ha/lib/master.rb @@ -45,11 +45,5 @@ mns.args = ["#{k8s['ip_part']}", "#{k8s['resources']['master']['ip_prefix']}", "#{i}", "#{k8s['cluster']['master']}", "#{k8s['resources']['master']['count']}"] end - subconfig.vm.provision "Restart VM", type: "shell" do |reboot| - reboot.privileged = true - reboot.inline = <<-SHELL - echo "----------------------------------|| Reboot to load all config" - SHELL - reboot.reboot = true - end + subconfig.vm.provision "Reboot to load all config", type:"shell", inline: "shutdown -r now" end diff --git a/kubernetes/ha/lib/node.rb b/kubernetes/ha/lib/node.rb index 86b7fda..9890084 100644 --- a/kubernetes/ha/lib/node.rb +++ b/kubernetes/ha/lib/node.rb @@ -42,11 +42,5 @@ ks.path = "script/bootstrap_node.sh" end - subconfig.vm.provision "Restart VM", type: "shell" do |reboot| - reboot.privileged = true - reboot.inline = <<-SHELL - echo "----------------------------------|| Reboot to load all config" - SHELL - reboot.reboot = true - end + subconfig.vm.provision "Reboot to load all config", type:"shell", inline: "shutdown -r now" end \ No newline at end of file diff --git a/kubernetes/ha/script/bootstrap.sh b/kubernetes/ha/script/bootstrap.sh index 91acaf9..605cb70 100644 --- a/kubernetes/ha/script/bootstrap.sh +++ b/kubernetes/ha/script/bootstrap.sh @@ -56,5 +56,4 @@ ufw enable <<> /etc/ssh/ssh_config -echo "ClientAliveCountMax 5" >> /etc/ssh/ssh_config \ No newline at end of file +echo -e "ClientAliveInterval 600\nTCPKeepAlive yes\nClientAliveCountMax 10" >> /etc/ssh/sshd_config diff --git a/kubernetes/ha/script/bootstrap_ha.sh b/kubernetes/ha/script/bootstrap_ha.sh index 07036ee..55ce2f6 100644 --- a/kubernetes/ha/script/bootstrap_ha.sh +++ b/kubernetes/ha/script/bootstrap_ha.sh @@ -61,5 +61,4 @@ ufw allow 6443/tcp ufw allow 22 # The SSH connection was unexpectedly closed by the remote end -echo "ClientAliveInterval 30" >> /etc/ssh/ssh_config -echo "ClientAliveCountMax 5" >> /etc/ssh/ssh_config \ No newline at end of file +echo -e "ClientAliveInterval 600\nTCPKeepAlive yes\nClientAliveCountMax 10" >> /etc/ssh/sshd_config