ansible/kubernetes-nodes.yaml

# This playbook sets up dependencies for the Kubernetes cluster nodes

---
- name: Reconfigure Kubernetes worker nodes
  hosts:
    - workers
    - nvr.kube.k-space.ee
  tasks:
    - name: Configure grub defaults
      copy:
        dest: "/etc/default/grub"
        content: |
          GRUB_DEFAULT=0
          GRUB_TIMEOUT_STYLE=countdown
          GRUB_TIMEOUT=5
          GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
          GRUB_CMDLINE_LINUX_DEFAULT="quiet splash memhp_default_state=online"
          GRUB_CMDLINE_LINUX="memhp_default_state=online rootflags=pquota"
      register: grub_defaults
      when: ansible_architecture == 'x86_64' and ansible_virtualization_role == 'guest' # TODO: detect xfs

    - name: Load grub defaults
      ansible.builtin.shell: update-grub
      when: grub_defaults.changed

    - name: Ensure nfs-common is installed
      ansible.builtin.apt:
        name: nfs-common
        state: present

- name: Reconfigure Kubernetes nodes
  hosts: kubernetes
  vars:
    KUBERNETES_VERSION: v1.30.3
    CRIO_VERSION: v1.30
    IP: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}"
  tasks:
    - name: Remove APT packages
      ansible.builtin.apt:
        name: "{{ item }}"
        state: absent
      loop:
        - kubelet
        - kubeadm
        - kubectl

    - name: Delete old repos
      ansible.builtin.shell: "rm /etc/apt/sources.list.d/kubernetes.list"
      changed_when: false
      failed_when: false

    - name: Delete old repos
      ansible.builtin.shell: "rm -rf /etc/apt/sources.list.d/devel*"
      changed_when: false
      failed_when: false

    - name: Add cri-o apt signing key
      ansible.builtin.apt_key:
        url: "https://pkgs.k8s.io/addons:/cri-o:/stable:/{{ CRIO_VERSION }}/deb/Release.key"
        state: present        

    - name: Add cri-o apt repo
      ansible.builtin.apt_repository:
        repo: "deb https://pkgs.k8s.io/addons:/cri-o:/stable:/{{ CRIO_VERSION }}/deb/ /"
        state: present        
        filename: crio

    - name: Install cri-o and conntrack
      ansible.builtin.apt:
        name: "{{ item }}"
        state: latest
      loop:
        - cri-o
        - conntrack  

    - name: Enable crio service
      ansible.builtin.systemd_service:
        name: crio
        enabled: yes

    - name: Download kubectl, kubeadm, kubelet
      ansible.builtin.get_url:
        url: "https://cdn.dl.k8s.io/release/{{ KUBERNETES_VERSION }}/bin/linux/{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}/{{ item }}"
        dest: "/usr/bin/{{ item }}-{{ KUBERNETES_VERSION }}"
        mode: '0755'
      loop:
        - kubelet
        - kubectl
        - kubeadm

    - name: Create symlinks for kubectl, kubeadm, kubelet
      ansible.builtin.file:
        src: "/usr/bin/{{ item }}-{{ KUBERNETES_VERSION }}"
        dest: "/usr/bin/{{ item }}"
        state: link
      loop:
        - kubelet
        - kubectl
        - kubeadm
      register: kubelet

    - name: Download calicoctl
      ansible.builtin.get_url:
        url: "https://github.com/projectcalico/calico/releases/download/v3.28.1/calicoctl-linux-{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}"
        dest: "/usr/bin/calicoctl"
        mode: '0755'

    - name: Install crictl
      ansible.builtin.shell: |
        VERSION="v1.31.1"
        wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-linux-amd64.tar.gz
        sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin
        rm -f crictl-$VERSION-linux-amd64.tar.gz

    - name: Create /etc/systemd/system/kubelet.service
      ansible.builtin.copy:
        content: |
          [Unit]
          Description=kubelet: The Kubernetes Node Agent
          Documentation=https://kubernetes.io/docs/home/
          Wants=network-online.target
          After=network-online.target
          [Service]
          ExecStart=/usr/bin/kubelet
          Restart=always
          StartLimitInterval=0
          RestartSec=10
          [Install]
          WantedBy=multi-user.target
        dest: /etc/systemd/system/kubelet.service
      register: kubelet_service

    - name: Create /etc/systemd/system/kubelet.service.d
      ansible.builtin.file:
        state: directory
        path: /etc/systemd/system/kubelet.service.d
      register: kubelet_service_directory

    - name: Create /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
      ansible.builtin.copy:
        content: |
          # Note: This dropin only works with kubeadm and kubelet v1.11+
          [Service]
          Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
          Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
          # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
          EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
          # This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
          # the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
          EnvironmentFile=-/etc/default/kubelet
          ExecStart=
          ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
        dest: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
      register: kubelet_service_directory_conf      

    - name: Enable kubelet service
      ansible.builtin.systemd_service:
        name: kubelet
        enabled: yes
        daemon_reload: true
      when: kubelet_service.changed or kubelet_service_directory.changed or kubelet_service_directory_conf.changed
      register: kubelet_service_reload

    - name: Restart Kubelet
      service:
        name: kubelet
        enabled: true
        state: restarted
      when: kubelet_service.changed or kubelet_service_reload.changed  

    - name: Reconfigure shutdownGracePeriod
      ansible.builtin.lineinfile:
        path: /var/lib/kubelet/config.yaml
        regexp: '^shutdownGracePeriod:'
        line: 'shutdownGracePeriod: 5m'
      ignore_errors: true

    - name: Reconfigure shutdownGracePeriodCriticalPods
      ansible.builtin.lineinfile:
        path: /var/lib/kubelet/config.yaml
        regexp: '^shutdownGracePeriodCriticalPods:'
        line: 'shutdownGracePeriodCriticalPods: 5m'
      ignore_errors: true

    - name: Work around unattended-upgrades
      ansible.builtin.lineinfile:
        path: /lib/systemd/logind.conf.d/unattended-upgrades-logind-maxdelay.conf
        regexp: '^InhibitDelayMaxSec='
        line: 'InhibitDelayMaxSec=5m0s'

    - name: Disable unneccesary services
      ignore_errors: true
      loop:
        - gdm3
        - snapd
        - bluetooth
        - multipathd
      service:
        name: "{{item}}"
        state: stopped
        enabled: false

    - name: Reset /etc/containers/registries.conf
      ansible.builtin.copy:
        content: "unqualified-search-registries = [\"docker.io\"]\n"
        dest: /etc/containers/registries.conf
      register: registries

    - name: Restart CRI-O
      service:
        name: cri-o
        state: restarted
      when: registries.changed

    - name: Reset /etc/modules
      ansible.builtin.copy:
        content: |
          overlay
          br_netfilter
        dest: /etc/modules
      register: kernel_modules
    - name: Load kernel modules
      ansible.builtin.shell: "cat /etc/modules | xargs -L 1 -t modprobe"
      when: kernel_modules.changed

    - name: Reset /etc/sysctl.d/99-k8s.conf
      ansible.builtin.copy:
        content: |
          net.ipv4.conf.all.accept_redirects  = 0
          net.bridge.bridge-nf-call-iptables  = 1
          net.ipv4.ip_forward                 = 1
          net.bridge.bridge-nf-call-ip6tables = 1
          vm.max_map_count                    = 524288
          fs.inotify.max_user_instances       = 1280
          fs.inotify.max_user_watches         = 655360
        dest: /etc/sysctl.d/99-k8s.conf
      register: sysctl

    - name: Reload sysctl config
      ansible.builtin.shell: "sysctl --system"
      when: sysctl.changed

    - name: Reconfigure kube-apiserver to use Passmower OIDC endpoint
      ansible.builtin.template:
        src: kube-apiserver.j2
        dest: /etc/kubernetes/manifests/kube-apiserver.yaml
        mode: 600
      register: apiserver
      when:
        - inventory_hostname in groups["masters"]

    - name: Restart kube-apiserver
      ansible.builtin.shell: "killall kube-apiserver"
      when: apiserver.changed