From 5e04a1bd43eb62ec33891a6307bbe63730f28b0e Mon Sep 17 00:00:00 2001 From: Erki Aas Date: Fri, 9 Aug 2024 12:07:03 +0300 Subject: [PATCH] provision new worker nodes with ansible --- ansible/kubernetes.yml | 134 ++++++++++++++++++++++++-------- ansible/templates/10-kubeadm.j2 | 12 +++ ansible/templates/kubelet.j2 | 43 ++++++++++ 3 files changed, 156 insertions(+), 33 deletions(-) create mode 100644 ansible/templates/10-kubeadm.j2 create mode 100644 ansible/templates/kubelet.j2 diff --git a/ansible/kubernetes.yml b/ansible/kubernetes.yml index 4e24bb0..7664e2e 100644 --- a/ansible/kubernetes.yml +++ b/ansible/kubernetes.yml @@ -1,4 +1,58 @@ --- +# ansible-galaxy install -r requirements.yaml +- name: Install cri-o + hosts: + - worker9.kube.k-space.ee + vars: + CRIO_VERSION: "v1.30" + tasks: + - name: ensure curl is installed + ansible.builtin.apt: + name: curl + state: present + + - name: Ensure /etc/apt/keyrings exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + + # TODO: fix + # - name: add k8s repo apt key + # ansible.builtin.shell: "curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/{{ CRIO_VERSION }}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg" + + - name: add k8s repo + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/{{ CRIO_VERSION }}/deb/ /" + state: present + filename: cri-o + + - name: check current crictl version + command: "/usr/bin/crictl --version" + failed_when: false + changed_when: false + register: crictl_version_check + + - name: download crictl + unarchive: + src: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ CRIO_VERSION }}/crictl-{{ CRIO_VERSION }}-linux-{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}.tar.gz" + dest: /tmp + remote_src: true + when: > + crictl_version_check.stdout is not defined or CRIO_VERSION not in crictl_version_check.stdout + register: crictl_download_check + + - name: move crictl binary into place + copy: + src: /tmp/crictl + dest: "/usr/bin/crictl" + when: > + exporter_download_check is changed + + - name: ensure crio is installed + ansible.builtin.apt: + name: cri-o + state: present + - name: Reconfigure Kubernetes worker nodes hosts: - storage @@ -40,7 +94,7 @@ loop: - kubelet - kubeadm - - kubectl + - kubectl - name: Download kubectl, kubeadm, kubelet ansible.builtin.get_url: @@ -52,6 +106,24 @@ - kubectl - kubeadm + - name: Create /etc/systemd/system/kubelet.service + ansible.builtin.copy: + content: | + [Unit] + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + Wants=network-online.target + After=network-online.target + [Service] + ExecStart=/usr/bin/kubelet + Restart=always + StartLimitInterval=0 + RestartSec=10 + [Install] + WantedBy=multi-user.target + dest: /etc/systemd/system/kubelet.service + register: kubelet_service + - name: Create symlinks for kubectl, kubeadm, kubelet ansible.builtin.file: src: "/usr/bin/{{ item }}-{{ KUBERNETES_VERSION }}" @@ -68,42 +140,32 @@ name: kubelet enabled: true state: restarted - when: kubelet.changed + daemon_reload: true + when: kubelet.changed or kubelet_service.changed - - name: Create /etc/systemd/system/kubelet.service - ansible.builtin.copy: - content: | - [Unit] - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - Wants=network-online.target - After=network-online.target - [Service] - ExecStart=/usr/local/bin/kubelet - Restart=always - StartLimitInterval=0 - RestartSec=10 - [Install] - WantedBy=multi-user.target - dest: /etc/systemd/system/kubelet.service + - name: Ensure /var/lib/kubelet exists + ansible.builtin.file: + path: /var/lib/kubelet + state: directory - - name: Reconfigure shutdownGracePeriod - ansible.builtin.lineinfile: - path: /var/lib/kubelet/config.yaml - regexp: '^shutdownGracePeriod:' - line: 'shutdownGracePeriod: 5m' + - name: Configure kubelet + ansible.builtin.template: + src: kubelet.j2 + dest: /var/lib/kubelet/config.yaml + mode: 644 - - name: Reconfigure shutdownGracePeriodCriticalPods - ansible.builtin.lineinfile: - path: /var/lib/kubelet/config.yaml - regexp: '^shutdownGracePeriodCriticalPods:' - line: 'shutdownGracePeriodCriticalPods: 5m' + - name: Ensure /etc/systemd/system/kubelet.service.d/ exists + ansible.builtin.file: + path: /etc/systemd/system/kubelet.service.d + state: directory - - name: Work around unattended-upgrades - ansible.builtin.lineinfile: - path: /lib/systemd/logind.conf.d/unattended-upgrades-logind-maxdelay.conf - regexp: '^InhibitDelayMaxSec=' - line: 'InhibitDelayMaxSec=5m0s' + - name: Configure kubelet service + ansible.builtin.template: + src: 10-kubeadm.j2 + dest: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + mode: 644 + + # TODO: register new node if needed - name: Disable unneccesary services ignore_errors: true @@ -112,11 +174,17 @@ - snapd - bluetooth - multipathd + - zram service: name: "{{item}}" state: stopped enabled: no + - name: Ensure /etc/containers exists + ansible.builtin.file: + path: /etc/containers + state: directory + - name: Reset /etc/containers/registries.conf ansible.builtin.copy: content: "unqualified-search-registries = [\"docker.io\"]\n" diff --git a/ansible/templates/10-kubeadm.j2 b/ansible/templates/10-kubeadm.j2 new file mode 100644 index 0000000..e63b2e9 --- /dev/null +++ b/ansible/templates/10-kubeadm.j2 @@ -0,0 +1,12 @@ +# Note: This dropin only works with kubeadm and kubelet v1.11+ +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use +# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +StandardOutput=null diff --git a/ansible/templates/kubelet.j2 b/ansible/templates/kubelet.j2 new file mode 100644 index 0000000..8a360f9 --- /dev/null +++ b/ansible/templates/kubelet.j2 @@ -0,0 +1,43 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s +cgroupDriver: systemd +clusterDNS: +- 10.96.0.10 +clusterDomain: cluster.local +cpuManagerReconcilePeriod: 0s +evictionPressureTransitionPeriod: 0s +fileCheckFrequency: 0s +healthzBindAddress: 127.0.0.1 +healthzPort: 10248 +httpCheckFrequency: 0s +imageMinimumGCAge: 0s +kind: KubeletConfiguration +logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 +memorySwap: {} +nodeStatusReportFrequency: 0s +nodeStatusUpdateFrequency: 0s +rotateCertificates: true +runtimeRequestTimeout: 0s +shutdownGracePeriod: 5m +shutdownGracePeriodCriticalPods: 5m +staticPodPath: /etc/kubernetes/manifests +streamingConnectionIdleTimeout: 0s +syncFrequency: 0s +volumeStatsAggPeriod: 0s