From 4d2071a5bd446cba7570bb5cd77a5fcb3054e5ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauri=20V=C3=B5sandi?= Date: Sun, 13 Aug 2023 20:21:15 +0300 Subject: [PATCH] Move Kubernetes cluster bootstrap partially to Ansible --- README.md | 58 +------------------------------------- ansible-kubernetes.yml | 63 ++++++++++++++++++++++++++++++++++++++++++ inventory.yml | 46 +++++++++++++++++------------- 3 files changed, 91 insertions(+), 76 deletions(-) create mode 100644 ansible-kubernetes.yml diff --git a/README.md b/README.md index 0ae9d98..f8f0f1f 100644 --- a/README.md +++ b/README.md @@ -160,30 +160,7 @@ Added some ARM64 workers by using Ubuntu 22.04 server on Raspberry Pi. After machines have booted up and you can reach them via SSH: -```bash -# Enable required kernel modules -cat > /etc/modules << EOF -overlay -br_netfilter -EOF -cat /etc/modules | xargs -L 1 -t modprobe - -# Finetune sysctl: -cat > /etc/sysctl.d/99-k8s.conf << EOF -net.ipv4.conf.all.accept_redirects = 0 -net.bridge.bridge-nf-call-iptables = 1 -net.ipv4.ip_forward = 1 -net.bridge.bridge-nf-call-ip6tables = 1 - -# Elasticsearch needs this -vm.max_map_count = 524288 - -# Bump inotify limits to make sure -fs.inotify.max_user_instances=1280 -fs.inotify.max_user_watches=655360 -EOF -sysctl --system - +``` # Disable Ubuntu caching DNS resolver systemctl disable systemd-resolved.service systemctl stop systemd-resolved @@ -206,39 +183,6 @@ apt-get install -yqq linux-image-generic apt-get remove -yq cloud-init linux-image-*-kvm ``` -Install packages: - -```bash -OS=xUbuntu_22.04 -VERSION=1.25 -echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /"| sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list -echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list - -rm -fv /etc/apt/trusted.gpg - -curl -s https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/libcontainers-archive-keyring.gpg -curl -s https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/libcontainers-crio-archive-keyring.gpg -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor > /etc/apt/trusted.gpg.d/packages-cloud-google.gpg - -echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list - -apt-get update -apt-get install -yqq --allow-change-held-packages apt-transport-https curl cri-o cri-o-runc kubelet=1.25.12-00 kubectl=1.25.12-00 kubeadm=1.25.12-00 cri-o=1.25.3~0 -apt-mark hold kubelet kubeadm kubectl cri-o - -cat << \EOF > /etc/containers/registries.conf -unqualified-search-registries = ["docker.io"] -# To pull Docker images from a mirror uncomment following -#[[registry]] -#prefix = "docker.io" -#location = "mirror.gcr.io" -EOF -sudo systemctl restart crio -sudo systemctl daemon-reload -sudo systemctl enable crio --now - -``` - On master: ``` diff --git a/ansible-kubernetes.yml b/ansible-kubernetes.yml new file mode 100644 index 0000000..39a6f15 --- /dev/null +++ b/ansible-kubernetes.yml @@ -0,0 +1,63 @@ +--- +- name: Pin kube components + hosts: kubernetes + tasks: + - name: Pin packages + loop: + - kubeadm + - kubectl + - kubelet + ansible.builtin.copy: + dest: "/etc/apt/preferences.d/{{ item }}" + content: | + Package: {{ item }} + Pin: version 1.26.* + Pin-Priority: 1001 + +- name: Reset /etc/containers/registries.conf + hosts: kubernetes + tasks: + - name: Copy /etc/containers/registries.conf + ansible.builtin.copy: + content: "unqualified-search-registries = [\"docker.io\"]\n" + dest: /etc/containers/registries.conf + register: registries + - name: Restart CRI-O + service: + name: cri-o + state: restarted + when: registries.changed + +- name: Reset /etc/modules + hosts: kubernetes + tasks: + - name: Copy /etc/modules + ansible.builtin.copy: + content: | + overlay + br_netfilter + dest: /etc/modules + register: kernel_modules + - name: Load kernel modules + ansible.builtin.shell: "cat /etc/modules | xargs -L 1 -t modprobe" + when: kernel_modules.changed + +- name: Reset /etc/sysctl.d/99-k8s.conf + hosts: kubernetes + tasks: + - name: Copy /etc/sysctl.d/99-k8s.conf + ansible.builtin.copy: + content: | + cat > /etc/sysctl.d/99-k8s.conf << EOF + net.ipv4.conf.all.accept_redirects = 0 + net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + vm.max_map_count = 524288 + fs.inotify.max_user_instances = 1280 + fs.inotify.max_user_watches = 655360 + dest: /etc/sysctl.d/99-k8s.conf + register: sysctl + - name: Reload sysctl config + ansible.builtin.shell: "sysctl --system" + when: sysctl.changed diff --git a/inventory.yml b/inventory.yml index ee93559..c5134b4 100644 --- a/inventory.yml +++ b/inventory.yml @@ -1,24 +1,32 @@ all: children: - masters: - hosts: - master1.kube.k-space.ee: - master2.kube.k-space.ee: - master3.kube.k-space.ee: - workers: - hosts: - mon1.kube.k-space.ee: - mon2.kube.k-space.ee: - mon3.kube.k-space.ee: - storage1.kube.k-space.ee: - storage2.kube.k-space.ee: - storage3.kube.k-space.ee: - storage4.kube.k-space.ee: - worker1.kube.k-space.ee: - worker2.kube.k-space.ee: - worker3.kube.k-space.ee: - worker4.kube.k-space.ee: - worker9.kube.k-space.ee: + kubernetes: + children: + masters: + hosts: + master1.kube.k-space.ee: + master2.kube.k-space.ee: + master3.kube.k-space.ee: + kubelets: + children: + mon: + hosts: + mon1.kube.k-space.ee: + mon2.kube.k-space.ee: + mon3.kube.k-space.ee: + storage: + hosts: + storage1.kube.k-space.ee: + storage2.kube.k-space.ee: + storage3.kube.k-space.ee: + storage4.kube.k-space.ee: + workers: + hosts: + worker1.kube.k-space.ee: + worker2.kube.k-space.ee: + worker3.kube.k-space.ee: + worker4.kube.k-space.ee: + worker9.kube.k-space.ee: doors: hosts: 100.102.3.1: