Add Proxmox network interfaces config

inventory.yaml

@@ -60,8 +60,6 @@ all:
     # Kubernetes cluster setup documented at
     # https://git.k-space.ee/k-space/kube
     kubernetes:
-      external.kube.k-space.ee:
-        ansible_host: 172.21.3.101
       children:
         kubelets:
           children:
@@ -83,6 +81,8 @@ all:
                   ansible_host: 172.21.3.83
                 worker4.kube.k-space.ee:
                   ansible_host: 172.21.3.84
+                external.kube.k-space.ee:
+                  ansible_host: 172.21.3.101
               children:
                 coralworkers:
                   hosts:

proxmox/ceph.yaml

@@ -2,11 +2,12 @@
 
 - name: configure ceph on proxmox
   hosts:
-    - pve90
+    - pve90.proxmox.infra.k-space.ee
-    - pve91
+    - pve91.proxmox.infra.k-space.ee
-    - pve92
+    - pve92.proxmox.infra.k-space.ee
-    - pve93
+    - pve93.proxmox.infra.k-space.ee
   gather_facts: false
+  serial: 1  # one host at a time
 
   tasks:
     - name: Load secrets

proxmox/general.yaml

@@ -1,17 +1,16 @@
 ---
 - name: PVE base configuration
   hosts: proxmox
-  tasks:
+  gather_facts: false
-    - name: Generate ipid
+  serial: 1  # one host at a time
-      set_fact:
-        new_fact: "{{ inventory_hostname | regex_replace('pve', '') }}"
 
+  tasks:
     - name: UI primary IP (/etc/hosts)
       tags: network
       ansible.builtin.lineinfile:
         path: /etc/hosts
-      regexp: ' {{ inventory_hostname_short }}$'
+        regexp: ' {{ inventory_hostname_short }}$'
-      line: '2001:bb8:4008:21:20::{{ ipid }} {{ inventory_hostname }} {{ inventory_hostname_short }}'
+        line: '{{ primary_ipv6 | split("/") | first }} {{ inventory_hostname }} {{ inventory_hostname_short }}'
 
     - name: /etc/network/interfaces
       tags: network

proxmox/group_vars/proxmox/main.yaml

@@ -0,0 +1,9 @@
+# Generate numeric id
+#   pve91.proxmox.infra.k-space.ee -> 91
+ipid: "{{ inventory_hostname_short | regex_replace('pve', '') }}"
+
+primary_ipv4: 172.21.20.{{ ipid }}/16
+primary_ipv4_gateway: 172.21.255.254
+primary_ipv6: 2001:bb8:4008:21:20::{{ ipid }}/64
+primary_ipv6_gateway: 2001:bb8:4008:21:ffff:ffff:ffff:fffe
+nfs_address: 2001:bb8:4008:101::{{ ipid }}/64

proxmox/host_vars/pve1.proxmox.infra.k-space.ee.yaml

@@ -0,0 +1,4 @@
+---
+bond_interfaces:
+  - eno49
+  - eno50

proxmox/host_vars/pve2.proxmox.infra.k-space.ee.yaml

@@ -0,0 +1,4 @@
+---
+bond_interfaces:
+  - eno49
+  - eno50

proxmox/host_vars/pve8.proxmox.infra.k-space.ee.yaml

@@ -0,0 +1,4 @@
+---
+bond_interfaces:
+  - eno49
+  - eno50

proxmox/host_vars/pve9.proxmox.infra.k-space.ee.yaml

@@ -0,0 +1,4 @@
+---
+bond_interfaces:
+  - eno49
+  - eno50

proxmox/host_vars/pve90.proxmox.infra.k-space.ee.yaml

@@ -6,4 +6,8 @@ ceph_mesh:
     - enp161s0
     - enp161s0d1
 
-bgp_router_id: 172.21.20.90    
+bgp_router_id: 172.21.20.90
+
+bond_interfaces:
+  - enp129s0f0np0
+  - enp129s0f1np1

proxmox/host_vars/pve91.proxmox.infra.k-space.ee.yaml

@@ -6,4 +6,8 @@ ceph_mesh:
     - enp161s0
     - enp161s0d1
 
-bgp_router_id: 172.21.20.91
+bgp_router_id: 172.21.20.91
+
+bond_interfaces:
+  - enp129s0f0np0
+  - enp129s0f1np1

proxmox/host_vars/pve92.proxmox.infra.k-space.ee.yaml

@@ -6,4 +6,8 @@ ceph_mesh:
     - enp161s0
     - enp161s0d1
 
-bgp_router_id: 172.21.20.92    
+bgp_router_id: 172.21.20.92
+
+bond_interfaces:
+  - enp130s0f0np0
+  - enp130s0f1np1

proxmox/host_vars/pve93.proxmox.infra.k-space.ee.yaml

@@ -6,4 +6,8 @@ ceph_mesh:
     - enp161s0
     - enp161s0d1
 
-bgp_router_id: 172.21.20.93    
+bgp_router_id: 172.21.20.93
+
+bond_interfaces:
+  - enp130s0f0np0
+  - enp130s0f1np1

proxmox/templates/network.interfaces.j2

@@ -0,0 +1,58 @@
+# {{ ansible_managed }}
+auto lo
+iface lo inet loopback
+
+# Slave interfaces
+{% for iface in bond_interfaces %}
+iface {{ iface }} inet manual
+{% endfor %}
+
+# Master interface
+auto bond0
+iface bond0 inet static
+  address 0.0.0.0
+  accept_ra 0
+  bond-mode 802.3ad
+  bond-slaves {{ bond_interfaces | join(' ') }}
+  bond-lacp-rate 1
+  bond-xmit-hash-policy layer2+3
+  bond-miimon 100
+  mtu 9000
+  post-up tc qdisc add dev bond0 root sfq perturb 10
+
+auto bond0.20
+iface bond0.20 inet static
+  address 0.0.0.0
+
+# Bridge for VM-s on VLAN20
+auto vmbr0
+iface vmbr0 inet static
+  address 0.0.0.0
+  accept_ra 0
+  mtu 1500
+  bridge_ports bond0.20
+  bridge-stp off
+  bridge-fd 0
+
+# Interface for NFS client
+auto bond0.101
+iface bond0.101 inet6 static
+  address {{ nfs_address }}
+  mtu 8996
+
+# Proxmox management interface on VLAN21
+auto vmbr1
+iface vmbr1 inet static
+  address {{ primary_ipv4 }}
+  gateway {{ primary_ipv4_gateway }}
+  up ip route add 193.40.103.36/30 via 172.21.3.81 dev vmbr1
+
+iface vmbr1 inet6 static
+  address {{ primary_ipv6 }}
+  gateway {{ primary_ipv6_gateway }}
+  mtu 8996
+  bridge_ports bond0.21
+  bridge-stp off
+  bridge-fd 0
+
+source /etc/network/interfaces.d/*