Route all of pvx9x over 40Gbit backplane

proxmox/ceph.yaml

@@ -7,6 +7,7 @@
     - pve92
     - pve93
   gather_facts: false
+
   tasks:
     - name: Load secrets
       ansible.builtin.include_vars:
@@ -52,9 +53,35 @@
         enabled: true
         state: started
 
+    - name: configure ebtables
+      ansible.builtin.template:
+        src: templates/ebtables.rules.j2
+        dest: /etc/ebtables.rules
+        mode: "u=rw,g=r,o=r"
+      notify: reload ebtables
+      tags: ebtables
+
+    - name: create ebtables systemd service
+      ansible.builtin.template:
+        src: templates/ebtables.service.j2
+        dest: /etc/systemd/system/ebtables.service
+        mode: "u=rw,g=r,o=r"
+      tags: ebtables
+
+    - name: enable/start ebtables service
+      ansible.builtin.systemd_service:
+        name: ebtables.service
+        enabled: true
+        state: started
+      tags: ebtables
 
   handlers:
     - name: reload FRR
       ansible.builtin.systemd_service:
         name: frr.service
-        state: reloaded
+        state: reloaded
+
+    - name: reload ebtables
+      ansible.builtin.systemd_service:
+        name: frr.service
+        state: restarted

proxmox/group_vars/proxmox/ebtables.yaml

@@ -0,0 +1,5 @@
+ebtables_broute_rules:
+  # Upgrade pve90-93 ipv6 traffic from Layer2 bridging to Layer3 routing
+  # So that Ceph traffic would always use 40Gbit backend network
+  # instead of 10Gbit public network
+  - '-A BROUTING -p IPv6 -i vmbr1 --ip6-dst 2001:bb8:4008:21:20::90/126 -j DROP'

proxmox/templates/ebtables.rules.j2

@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+*broute
+:BROUTING ACCEPT
+{% for rule in ebtables_broute_rules %}
+{{ rule }}
+{% endfor %}
+
+*filter
+:INPUT ACCEPT
+:FORWARD ACCEPT
+:OUTPUT ACCEPT
+

proxmox/templates/ebtables.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=ebtables persistent configuration
+DefaultDependencies=no
+Wants=network-pre.target systemd-modules-load.service local-fs.target
+Before=network-pre.target shutdown.target
+After=systemd-modules-load.service local-fs.target
+Conflicts=shutdown.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/bin/sh -c '/usr/sbin/ebtables-restore < /etc/ebtables.rules'
+
+[Install]
+WantedBy=multi-user.target

proxmox/templates/frr.conf.j2

@@ -25,6 +25,7 @@ exit
 !
 router openfabric 1
  net {{ ceph_mesh.openfabric_net }}
+ redistribute ipv6 local
  lsp-gen-interval 5
 exit
 !