Move Kubernetes cluster bootstrap partially to Ansible

This commit is contained in:
Lauri Võsandi 2023-08-13 20:21:15 +03:00
parent ecf9111f8f
commit 4d2071a5bd
3 changed files with 91 additions and 76 deletions

View File

@ -160,30 +160,7 @@ Added some ARM64 workers by using Ubuntu 22.04 server on Raspberry Pi.
After machines have booted up and you can reach them via SSH:
```bash
# Enable required kernel modules
cat > /etc/modules << EOF
overlay
br_netfilter
EOF
cat /etc/modules | xargs -L 1 -t modprobe
# Finetune sysctl:
cat > /etc/sysctl.d/99-k8s.conf << EOF
net.ipv4.conf.all.accept_redirects = 0
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
# Elasticsearch needs this
vm.max_map_count = 524288
# Bump inotify limits to make sure
fs.inotify.max_user_instances=1280
fs.inotify.max_user_watches=655360
EOF
sysctl --system
```
# Disable Ubuntu caching DNS resolver
systemctl disable systemd-resolved.service
systemctl stop systemd-resolved
@ -206,39 +183,6 @@ apt-get install -yqq linux-image-generic
apt-get remove -yq cloud-init linux-image-*-kvm
```
Install packages:
```bash
OS=xUbuntu_22.04
VERSION=1.25
echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /"| sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
rm -fv /etc/apt/trusted.gpg
curl -s https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/libcontainers-archive-keyring.gpg
curl -s https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | gpg --dearmor > /etc/apt/trusted.gpg.d/libcontainers-crio-archive-keyring.gpg
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor > /etc/apt/trusted.gpg.d/packages-cloud-google.gpg
echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list
apt-get update
apt-get install -yqq --allow-change-held-packages apt-transport-https curl cri-o cri-o-runc kubelet=1.25.12-00 kubectl=1.25.12-00 kubeadm=1.25.12-00 cri-o=1.25.3~0
apt-mark hold kubelet kubeadm kubectl cri-o
cat << \EOF > /etc/containers/registries.conf
unqualified-search-registries = ["docker.io"]
# To pull Docker images from a mirror uncomment following
#[[registry]]
#prefix = "docker.io"
#location = "mirror.gcr.io"
EOF
sudo systemctl restart crio
sudo systemctl daemon-reload
sudo systemctl enable crio --now
```
On master:
```

63
ansible-kubernetes.yml Normal file
View File

@ -0,0 +1,63 @@
---
- name: Pin kube components
hosts: kubernetes
tasks:
- name: Pin packages
loop:
- kubeadm
- kubectl
- kubelet
ansible.builtin.copy:
dest: "/etc/apt/preferences.d/{{ item }}"
content: |
Package: {{ item }}
Pin: version 1.26.*
Pin-Priority: 1001
- name: Reset /etc/containers/registries.conf
hosts: kubernetes
tasks:
- name: Copy /etc/containers/registries.conf
ansible.builtin.copy:
content: "unqualified-search-registries = [\"docker.io\"]\n"
dest: /etc/containers/registries.conf
register: registries
- name: Restart CRI-O
service:
name: cri-o
state: restarted
when: registries.changed
- name: Reset /etc/modules
hosts: kubernetes
tasks:
- name: Copy /etc/modules
ansible.builtin.copy:
content: |
overlay
br_netfilter
dest: /etc/modules
register: kernel_modules
- name: Load kernel modules
ansible.builtin.shell: "cat /etc/modules | xargs -L 1 -t modprobe"
when: kernel_modules.changed
- name: Reset /etc/sysctl.d/99-k8s.conf
hosts: kubernetes
tasks:
- name: Copy /etc/sysctl.d/99-k8s.conf
ansible.builtin.copy:
content: |
cat > /etc/sysctl.d/99-k8s.conf << EOF
net.ipv4.conf.all.accept_redirects = 0
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
vm.max_map_count = 524288
fs.inotify.max_user_instances = 1280
fs.inotify.max_user_watches = 655360
dest: /etc/sysctl.d/99-k8s.conf
register: sysctl
- name: Reload sysctl config
ansible.builtin.shell: "sysctl --system"
when: sysctl.changed

View File

@ -1,24 +1,32 @@
all:
children:
masters:
hosts:
master1.kube.k-space.ee:
master2.kube.k-space.ee:
master3.kube.k-space.ee:
workers:
hosts:
mon1.kube.k-space.ee:
mon2.kube.k-space.ee:
mon3.kube.k-space.ee:
storage1.kube.k-space.ee:
storage2.kube.k-space.ee:
storage3.kube.k-space.ee:
storage4.kube.k-space.ee:
worker1.kube.k-space.ee:
worker2.kube.k-space.ee:
worker3.kube.k-space.ee:
worker4.kube.k-space.ee:
worker9.kube.k-space.ee:
kubernetes:
children:
masters:
hosts:
master1.kube.k-space.ee:
master2.kube.k-space.ee:
master3.kube.k-space.ee:
kubelets:
children:
mon:
hosts:
mon1.kube.k-space.ee:
mon2.kube.k-space.ee:
mon3.kube.k-space.ee:
storage:
hosts:
storage1.kube.k-space.ee:
storage2.kube.k-space.ee:
storage3.kube.k-space.ee:
storage4.kube.k-space.ee:
workers:
hosts:
worker1.kube.k-space.ee:
worker2.kube.k-space.ee:
worker3.kube.k-space.ee:
worker4.kube.k-space.ee:
worker9.kube.k-space.ee:
doors:
hosts:
100.102.3.1: