migrate workers to infra vlan, use bgp for calico, use calico for lb service annoucements

tigera-operator/README.md

@@ -2,7 +2,7 @@
 
 Calico implements
 [container network interface plugin](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/)
-which enables pods to talk to eachother.
+which enables inter-pod network with BGP, also advertising Service LB IPs.
 
 # For user
 
@@ -13,7 +13,8 @@ Nothing specific to point out, this is standard Kubernetes feature
 Tigera operator was used to deploy Calico:
 
 ```
-curl https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/tigera-operator.yaml -O
+curl https://raw.githubusercontent.com/projectcalico/calico/v3.28.1/manifests/tigera-operator.yaml -O
 kubectl apply --server-side --force-conflicts -f tigera-operator.yaml
 kubectl apply -f application.yaml
+kubectl -n calico-system create secret generic bgp-secrets --from-literal=password=...
 ```

tigera-operator/application.yaml

@@ -12,8 +12,8 @@ spec:
     - blockSize: 26
       cidr: 10.244.0.0/16
       encapsulation: VXLANCrossSubnet
-      natOutgoing: Enabled
-      nodeSelector: all()
+      natOutgoing: Disabled
+      nodeSelector: all() 
 ---
 # This section configures the Calico API server.
 # For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer
@@ -22,3 +22,60 @@ kind: APIServer
 metadata:
   name: default
 spec: {}
+---
+apiVersion: crd.projectcalico.org/v1
+kind: BGPPeer
+metadata:
+  name: mikrotik-router
+spec:
+  peerIP: 172.21.255.254
+  asNumber: 64567
+  keepOriginalNextHop: true
+  password:
+    secretKeyRef:
+      name: bgp-secrets
+      key: password
+---
+apiVersion: crd.projectcalico.org/v1
+kind: BGPConfiguration
+metadata:
+  name: default
+spec:
+  serviceLoadBalancerIPs:
+  - cidr: 172.21.51.4/32
+  - cidr: 172.21.53.1/32
+  - cidr: 172.21.53.2/32  
+  - cidr: 172.21.53.3/32  
+  - cidr: 193.40.103.36/32
+  - cidr: 193.40.103.37/32
+  - cidr: 193.40.103.38/32    
+  - cidr: 193.40.103.39/32
+  - cidr: 62.65.250.36/32
+  - cidr: 62.65.250.37/32
+  - cidr: 62.65.250.2/32
+  - cidr: 193.40.103.25/32
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-access
+  namespace: calico-system
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  resourceNames: ["bgp-secrets"]
+  verbs: ["watch", "list", "get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: secret-access
+  namespace: calico-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: secret-access
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: calico-system