provision new worker nodes with ansible

ansible/kubernetes.yml

@@ -1,4 +1,58 @@
 ---
+# ansible-galaxy install -r requirements.yaml
+- name: Install cri-o
+  hosts:
+    - worker9.kube.k-space.ee
+  vars: 
+    CRIO_VERSION: "v1.30"
+  tasks:
+    - name: ensure curl is installed
+      ansible.builtin.apt:
+        name: curl
+        state: present
+
+    - name: Ensure /etc/apt/keyrings exists
+      ansible.builtin.file:
+        path: /etc/apt/keyrings
+        state: directory
+
+    # TODO: fix
+    # - name: add k8s repo apt key
+    #   ansible.builtin.shell: "curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/{{ CRIO_VERSION }}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg"
+    
+    - name: add k8s repo
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/{{ CRIO_VERSION }}/deb/ /" 
+        state: present
+        filename: cri-o
+
+    - name: check current crictl version
+      command: "/usr/bin/crictl --version"
+      failed_when: false
+      changed_when: false
+      register: crictl_version_check
+
+    - name: download crictl
+      unarchive:
+        src: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ CRIO_VERSION }}/crictl-{{ CRIO_VERSION }}-linux-{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}.tar.gz"
+        dest: /tmp
+        remote_src: true
+      when: >
+        crictl_version_check.stdout is not defined or CRIO_VERSION not in crictl_version_check.stdout
+      register: crictl_download_check
+
+    - name: move crictl binary into place
+      copy:
+        src: /tmp/crictl
+        dest: "/usr/bin/crictl"
+      when: >
+        exporter_download_check is changed
+
+    - name: ensure crio is installed
+      ansible.builtin.apt:
+        name: cri-o
+        state: present
+
 - name: Reconfigure Kubernetes worker nodes
   hosts:
     - storage
@@ -40,7 +94,7 @@
       loop:
         - kubelet
         - kubeadm
-        - kubectl
+        - kubectl 
 
     - name: Download kubectl, kubeadm, kubelet
       ansible.builtin.get_url:
@@ -52,6 +106,24 @@
         - kubectl
         - kubeadm
 
+    - name: Create /etc/systemd/system/kubelet.service
+      ansible.builtin.copy:
+        content: |
+          [Unit]
+          Description=kubelet: The Kubernetes Node Agent
+          Documentation=https://kubernetes.io/docs/home/
+          Wants=network-online.target
+          After=network-online.target
+          [Service]
+          ExecStart=/usr/bin/kubelet
+          Restart=always
+          StartLimitInterval=0
+          RestartSec=10
+          [Install]
+          WantedBy=multi-user.target
+        dest: /etc/systemd/system/kubelet.service
+      register: kubelet_service        
+
     - name: Create symlinks for kubectl, kubeadm, kubelet
       ansible.builtin.file:
         src: "/usr/bin/{{ item }}-{{ KUBERNETES_VERSION }}"
@@ -68,42 +140,32 @@
         name: kubelet
         enabled: true
         state: restarted
-      when: kubelet.changed
+        daemon_reload: true
+      when: kubelet.changed or kubelet_service.changed
 
-    - name: Create /etc/systemd/system/kubelet.service
-      ansible.builtin.copy:
-        content: |
-          [Unit]
-          Description=kubelet: The Kubernetes Node Agent
-          Documentation=https://kubernetes.io/docs/home/
-          Wants=network-online.target
-          After=network-online.target
-          [Service]
-          ExecStart=/usr/local/bin/kubelet
-          Restart=always
-          StartLimitInterval=0
-          RestartSec=10
-          [Install]
-          WantedBy=multi-user.target
-        dest: /etc/systemd/system/kubelet.service
+    - name: Ensure /var/lib/kubelet exists
+      ansible.builtin.file:
+        path: /var/lib/kubelet
+        state: directory
 
-    - name: Reconfigure shutdownGracePeriod
-      ansible.builtin.lineinfile:
-        path: /var/lib/kubelet/config.yaml
-        regexp: '^shutdownGracePeriod:'
-        line: 'shutdownGracePeriod: 5m'
+    - name: Configure kubelet
+      ansible.builtin.template:
+        src: kubelet.j2
+        dest: /var/lib/kubelet/config.yaml
+        mode: 644
 
-    - name: Reconfigure shutdownGracePeriodCriticalPods
-      ansible.builtin.lineinfile:
-        path: /var/lib/kubelet/config.yaml
-        regexp: '^shutdownGracePeriodCriticalPods:'
-        line: 'shutdownGracePeriodCriticalPods: 5m'
+    - name: Ensure /etc/systemd/system/kubelet.service.d/ exists
+      ansible.builtin.file:
+        path: /etc/systemd/system/kubelet.service.d
+        state: directory
 
-    - name: Work around unattended-upgrades
-      ansible.builtin.lineinfile:
-        path: /lib/systemd/logind.conf.d/unattended-upgrades-logind-maxdelay.conf
-        regexp: '^InhibitDelayMaxSec='
-        line: 'InhibitDelayMaxSec=5m0s'
+    - name: Configure kubelet service
+      ansible.builtin.template:
+        src: 10-kubeadm.j2
+        dest: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+        mode: 644
+
+    # TODO: register new node if needed
 
     - name: Disable unneccesary services
       ignore_errors: true
@@ -112,11 +174,17 @@
         - snapd
         - bluetooth
         - multipathd
+        - zram
       service:
         name: "{{item}}"
         state: stopped
         enabled: no
 
+    - name: Ensure /etc/containers exists
+      ansible.builtin.file:
+        path: /etc/containers
+        state: directory
+
     - name: Reset /etc/containers/registries.conf
       ansible.builtin.copy:
         content: "unqualified-search-registries = [\"docker.io\"]\n"

ansible/templates/10-kubeadm.j2

@@ -0,0 +1,12 @@
+# Note: This dropin only works with kubeadm and kubelet v1.11+
+[Service]
+Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
+Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
+# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
+EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
+# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
+# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
+EnvironmentFile=-/etc/default/kubelet
+ExecStart=
+ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
+StandardOutput=null

ansible/templates/kubelet.j2

@@ -0,0 +1,43 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    cacheTTL: 0s
+    enabled: true
+  x509:
+    clientCAFile: /etc/kubernetes/pki/ca.crt
+authorization:
+  mode: Webhook
+  webhook:
+    cacheAuthorizedTTL: 0s
+    cacheUnauthorizedTTL: 0s
+cgroupDriver: systemd
+clusterDNS:
+- 10.96.0.10
+clusterDomain: cluster.local
+cpuManagerReconcilePeriod: 0s
+evictionPressureTransitionPeriod: 0s
+fileCheckFrequency: 0s
+healthzBindAddress: 127.0.0.1
+healthzPort: 10248
+httpCheckFrequency: 0s
+imageMinimumGCAge: 0s
+kind: KubeletConfiguration
+logging:
+  flushFrequency: 0
+  options:
+    json:
+      infoBufferSize: "0"
+  verbosity: 0
+memorySwap: {}
+nodeStatusReportFrequency: 0s
+nodeStatusUpdateFrequency: 0s
+rotateCertificates: true
+runtimeRequestTimeout: 0s
+shutdownGracePeriod: 5m
+shutdownGracePeriodCriticalPods: 5m
+staticPodPath: /etc/kubernetes/manifests
+streamingConnectionIdleTimeout: 0s
+syncFrequency: 0s
+volumeStatsAggPeriod: 0s