migrate workers to infra vlan, use bgp for calico, use calico for lb service annoucements

bind/README.md

@@ -26,6 +26,13 @@ To find usage examples in this repository use
 
 
 # For administrator
+Ingresses and DNSEndpoints referring to `k-space.ee`, `kspace.ee`, `k6.ee`
+are picked up automatically by `external-dns` and updated on primary.
+
+The primary triggers notification events to `172.21.53.{1..3}`
+which are internally exposed IP-s of the secondaries.
+
+# Secrets
 
 To configure TSIG secrets:
 
@@ -61,7 +68,7 @@ zone "foobar.com" {
     file "/var/lib/bind/db.foobar.com";
     allow-update { !rejected; key foobar; };
     allow-transfer { !rejected; key readonly; key foobar; };
-    notify explicit; also-notify { 172.20.53.1; 172.20.53.2; 172.20.53.3; };
+    notify explicit; also-notify { 172.21.53.1; 172.21.53.2; 172.21.53.3; };
 };
 ```
 

bind/bind-secondary.yaml

@@ -116,7 +116,7 @@ metadata:
 spec:
   type: LoadBalancer
   externalTrafficPolicy: Local
-  loadBalancerIP: 172.20.53.1
+  loadBalancerIP: 172.21.53.1
   selector:
     app: bind-secondary
     statefulset.kubernetes.io/pod-name: bind-secondary-0
@@ -138,7 +138,7 @@ metadata:
 spec:
   type: LoadBalancer
   externalTrafficPolicy: Local
-  loadBalancerIP: 172.20.53.2
+  loadBalancerIP: 172.21.53.2
   selector:
     app: bind-secondary
     statefulset.kubernetes.io/pod-name: bind-secondary-1
@@ -160,7 +160,7 @@ metadata:
 spec:
   type: LoadBalancer
   externalTrafficPolicy: Local
-  loadBalancerIP: 172.20.53.3
+  loadBalancerIP: 172.21.53.3
   selector:
     app: bind-secondary
     statefulset.kubernetes.io/pod-name: bind-secondary-2

elastic-system/application.yml

@@ -150,7 +150,7 @@ metadata:
 spec:
   type: LoadBalancer
   externalTrafficPolicy: Local
-  loadBalancerIP: 172.20.51.4
+  loadBalancerIP: 172.21.51.4
   ports:
     - name: filebeat-syslog
       port: 514
@@ -169,7 +169,7 @@ metadata:
 spec:
   type: LoadBalancer
   externalTrafficPolicy: Local
-  loadBalancerIP: 172.20.51.4
+  loadBalancerIP: 172.21.51.4
   ports:
     - name: filebeat-syslog
       port: 514

metallb-system/application.yml

@@ -26,19 +26,7 @@ metadata:
   namespace: metallb-system
 spec:
   addresses:
-  - 172.20.51.0/24
+  - 172.21.51.0/24
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: zoo
-  namespace: metallb-system
-spec:
-  ipAddressPools:
-    - zoo
-    - bind-secondary-external
-    - bind-secondary-internal
-    - wildduck
 ---
 # Slice of public EEnet subnet using MetalLB L3 method
 apiVersion: metallb.io/v1beta1
@@ -67,7 +55,7 @@ metadata:
   namespace: metallb-system
 spec:
   addresses:
-  - 172.20.53.0/24
+  - 172.21.53.0/24
 ---
 apiVersion: metallb.io/v1beta1
 kind: IPAddressPool
@@ -99,13 +87,3 @@ spec:
   passwordSecret:
     name: mikrotik-router
     namespace: metallb-system
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
-  name: public
-  namespace: metallb-system
-spec:
-  ipAddressPools:
-  - eenet
-  - elisa

monitoring/blackbox-exporter.yaml

@@ -33,9 +33,9 @@ spec:
       static:
         - 193.40.103.2
         - 62.65.250.2
-        - 172.20.53.1
+        - 172.21.53.1
-        - 172.20.53.2
+        - 172.21.53.2
-        - 172.20.53.3
+        - 172.21.53.3
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: Probe

tigera-operator/README.md

@@ -2,7 +2,7 @@
 
 Calico implements
 [container network interface plugin](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/)
-which enables pods to talk to eachother.
+which enables inter-pod network with BGP, also advertising Service LB IPs.
 
 # For user
 
@@ -13,7 +13,8 @@ Nothing specific to point out, this is standard Kubernetes feature
 Tigera operator was used to deploy Calico:
 
 ```
-curl https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/tigera-operator.yaml -O
+curl https://raw.githubusercontent.com/projectcalico/calico/v3.28.1/manifests/tigera-operator.yaml -O
 kubectl apply --server-side --force-conflicts -f tigera-operator.yaml
 kubectl apply -f application.yaml
+kubectl -n calico-system create secret generic bgp-secrets --from-literal=password=...
 ```

tigera-operator/application.yaml

@@ -12,7 +12,7 @@ spec:
     - blockSize: 26
       cidr: 10.244.0.0/16
       encapsulation: VXLANCrossSubnet
-      natOutgoing: Enabled
+      natOutgoing: Disabled
       nodeSelector: all() 
 ---
 # This section configures the Calico API server.
@@ -22,3 +22,60 @@ kind: APIServer
 metadata:
   name: default
 spec: {}
+---
+apiVersion: crd.projectcalico.org/v1
+kind: BGPPeer
+metadata:
+  name: mikrotik-router
+spec:
+  peerIP: 172.21.255.254
+  asNumber: 64567
+  keepOriginalNextHop: true
+  password:
+    secretKeyRef:
+      name: bgp-secrets
+      key: password
+---
+apiVersion: crd.projectcalico.org/v1
+kind: BGPConfiguration
+metadata:
+  name: default
+spec:
+  serviceLoadBalancerIPs:
+  - cidr: 172.21.51.4/32
+  - cidr: 172.21.53.1/32
+  - cidr: 172.21.53.2/32  
+  - cidr: 172.21.53.3/32  
+  - cidr: 193.40.103.36/32
+  - cidr: 193.40.103.37/32
+  - cidr: 193.40.103.38/32    
+  - cidr: 193.40.103.39/32
+  - cidr: 62.65.250.36/32
+  - cidr: 62.65.250.37/32
+  - cidr: 62.65.250.2/32
+  - cidr: 193.40.103.25/32
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-access
+  namespace: calico-system
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  resourceNames: ["bgp-secrets"]
+  verbs: ["watch", "list", "get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: secret-access
+  namespace: calico-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: secret-access
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: calico-system