From 5643f1eec1b432040e549a31878f6b31b50df18d Mon Sep 17 00:00:00 2001 From: Arti Zirk Date: Thu, 31 Jul 2025 00:12:27 +0300 Subject: [PATCH] Add Proxmox Ceph mesh network playbook --- inventory.yaml | 8 ++++ proxmox/README.md | 58 ++++++++++++++++++++++++++++ proxmox/ceph.yaml | 49 +++++++++++++++++++++++ proxmox/host_vars/pve90.yaml | 7 ++++ proxmox/host_vars/pve91.yaml | 7 ++++ proxmox/host_vars/pve92.yaml | 7 ++++ proxmox/host_vars/pve93.yaml | 7 ++++ proxmox/templates/ceph.interfaces.j2 | 15 +++++++ proxmox/templates/frr.conf.j2 | 30 ++++++++++++++ 9 files changed, 188 insertions(+) create mode 100644 proxmox/README.md create mode 100644 proxmox/ceph.yaml create mode 100644 proxmox/host_vars/pve90.yaml create mode 100644 proxmox/host_vars/pve91.yaml create mode 100644 proxmox/host_vars/pve92.yaml create mode 100644 proxmox/host_vars/pve93.yaml create mode 100644 proxmox/templates/ceph.interfaces.j2 create mode 100644 proxmox/templates/frr.conf.j2 diff --git a/inventory.yaml b/inventory.yaml index 0a8c220..88478f5 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -46,6 +46,14 @@ all: ansible_host: 172.21.20.8 pve9: ansible_host: 172.21.20.9 + pve90: + ansible_host: 172.21.20.90 + pve91: + ansible_host: 172.21.20.91 +# pve92: +# ansible_host: 172.21.20.92 + pve93: + ansible_host: 172.21.20.93 # Kubernetes cluster setup documented at # https://git.k-space.ee/k-space/kube diff --git a/proxmox/README.md b/proxmox/README.md new file mode 100644 index 0000000..80859b4 --- /dev/null +++ b/proxmox/README.md @@ -0,0 +1,58 @@ +# Proxmox Virtual Environment + +## K-Space Hyper Converged CEPH setup + +1. Configure a mesh network + + ansible-playbook proxmox/ceph.yaml + + This will configure the 40Gbit interfaces and FRR daemon with OpenFabric routing. + Our CEPH setup uses a private IPv6 subnet for inner cluster communication. + + fdcc:a182:4fed::/64 +2. Setup CEPH packages on all nodes + + pveceph install --repository no-subscription --version squid +3. CEPH init + + pveceph init --network fdcc:a182:4fed::/64 +4. Create CEPH monitors on each node + + pveceph mon create +5. Also create CEPH managers on each node + + pveceph mgr create +6. Create OSD daemons for each disk on all nodes + + NVMe drives will get 2 OSD daemons per disk for better IOPS + + pveceph osd create /dev/nvme0n1 --crush-device-class nvme --osds-per-device 2 + + HDD-s will get just 1 + + pveceph osd create /dev/sdX --crush-device-class hdd +7. Create CRUSH Maps + + We want to separate out HDD and NVMe storage into different storage buckets. + + Default `replicated_rule` would put datablock on all of the available disks + + # ceph osd crush rule create-replicated + ceph osd crush rule create-replicated replicated_nvme default host nvme + ceph osd crush rule create-replicated replicated_hdd default host hdd +8. Create CEPH Pools for VM disk images + + This is done in individual node Ceph -> Pools configuration + + **NB:** Under advanced, select correct Crush Rule (nvme or hdd) + +9. Create CephFS Storage pool for ISO images + + First create metadata server on each node + + pveceph mds create + + Then on one of the individual nodes create a CephFS. + + After that is done you can modify under Pools change the cephfs_data and cephfs_metadata + Crush rules to use NVMe drives. diff --git a/proxmox/ceph.yaml b/proxmox/ceph.yaml new file mode 100644 index 0000000..85f3575 --- /dev/null +++ b/proxmox/ceph.yaml @@ -0,0 +1,49 @@ +--- + +- name: configure ceph on proxmox + hosts: + - pve90 + - pve91 + # - pve92 + - pve93 + gather_facts: false + tasks: + - name: configure mesh network + ansible.builtin.template: + src: templates/ceph.interfaces.j2 + dest: /etc/network/interfaces.d/ceph + tags: network + + - name: ifup lo + ansible.builtin.command: + cmd: ifup lo + tags: network + + - name: ifup mesh interfaces + ansible.builtin.command: + cmd: "ifup {{ item }}" + loop: "{{ ceph_mesh.interfaces }}" + loop_control: + label: "ifup {{ item }}" + tags: network + + - name: enable fabricd OpenFabric in FRR + ansible.builtin.lineinfile: + path: /etc/frr/daemons + regexp: ^fabricd=.*$ + line: fabricd=yes + notify: reload FRR + tags: frr + + - name: configure FRR + ansible.builtin.template: + src: templates/frr.conf.j2 + dest: /etc/frr/frr.conf + notify: reload FRR + tags: frr + + handlers: + - name: reload FRR + ansible.builtin.systemd_service: + name: frr.service + state: reloaded \ No newline at end of file diff --git a/proxmox/host_vars/pve90.yaml b/proxmox/host_vars/pve90.yaml new file mode 100644 index 0000000..5c23313 --- /dev/null +++ b/proxmox/host_vars/pve90.yaml @@ -0,0 +1,7 @@ +--- +ceph_mesh: + address: fdcc:a182:4fed::90/128 + openfabric_net: 49.0000.0000.0090.00 + interfaces: + - enp161s0 + - enp161s0d1 \ No newline at end of file diff --git a/proxmox/host_vars/pve91.yaml b/proxmox/host_vars/pve91.yaml new file mode 100644 index 0000000..8d1053d --- /dev/null +++ b/proxmox/host_vars/pve91.yaml @@ -0,0 +1,7 @@ +--- +ceph_mesh: + address: fdcc:a182:4fed::91/128 + openfabric_net: 49.0000.0000.0091.00 + interfaces: + - enp161s0 + - enp161s0d1 \ No newline at end of file diff --git a/proxmox/host_vars/pve92.yaml b/proxmox/host_vars/pve92.yaml new file mode 100644 index 0000000..def891d --- /dev/null +++ b/proxmox/host_vars/pve92.yaml @@ -0,0 +1,7 @@ +--- +ceph_mesh: + address: fdcc:a182:4fed::92/128 + openfabric_net: 49.0000.0000.0092.00 + interfaces: + - enp161s0 + - enp161s0d1 \ No newline at end of file diff --git a/proxmox/host_vars/pve93.yaml b/proxmox/host_vars/pve93.yaml new file mode 100644 index 0000000..d2b5a59 --- /dev/null +++ b/proxmox/host_vars/pve93.yaml @@ -0,0 +1,7 @@ +--- +ceph_mesh: + address: fdcc:a182:4fed::93/128 + openfabric_net: 49.0000.0000.0093.00 + interfaces: + - enp161s0 + - enp161s0d1 \ No newline at end of file diff --git a/proxmox/templates/ceph.interfaces.j2 b/proxmox/templates/ceph.interfaces.j2 new file mode 100644 index 0000000..23ce0d7 --- /dev/null +++ b/proxmox/templates/ceph.interfaces.j2 @@ -0,0 +1,15 @@ +# {{ ansible_managed }} +# ifupdown interfaces(5) file for setting up CEPH network + +# Real routing is handled by FRR routing daemon +auto lo +iface lo inet loopback + up ip -6 addr add {{ ceph_mesh.address }} dev lo + +{% for iface in ceph_mesh.interfaces %} +auto {{ iface }} +iface {{ iface }} inet6 static + pre-up ip link set $IFACE up + mtu 9000 + +{% endfor %} \ No newline at end of file diff --git a/proxmox/templates/frr.conf.j2 b/proxmox/templates/frr.conf.j2 new file mode 100644 index 0000000..621c128 --- /dev/null +++ b/proxmox/templates/frr.conf.j2 @@ -0,0 +1,30 @@ +# {{ ansible_managed }} +frr version 10.2.2 +frr defaults traditional +hostname {{ inventory_hostname }} +log syslog informational +no ip forwarding +service integrated-vtysh-config +! +{% for iface in ceph_mesh.interfaces %} +interface {{ iface }} + ipv6 router openfabric 1 + openfabric csnp-interval 5 + openfabric hello-interval 1 + openfabric hello-multiplier 3 +exit +! +{% endfor %} +interface lo + ipv6 router openfabric 1 + openfabric csnp-interval 5 + openfabric hello-interval 1 + openfabric hello-multiplier 3 + openfabric passive +exit +! +router openfabric 1 + net {{ ceph_mesh.openfabric_net }} + lsp-gen-interval 5 +exit +!