From 9bf5e2408a4bb6809df9009b381bb4a55ee5e803 Mon Sep 17 00:00:00 2001 From: Erki Aas Date: Wed, 14 Aug 2024 18:15:04 +0300 Subject: [PATCH] migrate workers to infra vlan, use bgp for calico, use calico for lb service annoucements --- bind/README.md | 9 ++++- bind/bind-secondary.yaml | 6 +-- elastic-system/application.yml | 4 +- metallb-system/application.yml | 26 +------------ monitoring/blackbox-exporter.yaml | 6 +-- tigera-operator/README.md | 5 ++- tigera-operator/application.yaml | 61 ++++++++++++++++++++++++++++++- 7 files changed, 80 insertions(+), 37 deletions(-) diff --git a/bind/README.md b/bind/README.md index ed7e45f..b08cd6c 100644 --- a/bind/README.md +++ b/bind/README.md @@ -26,6 +26,13 @@ To find usage examples in this repository use # For administrator +Ingresses and DNSEndpoints referring to `k-space.ee`, `kspace.ee`, `k6.ee` +are picked up automatically by `external-dns` and updated on primary. + +The primary triggers notification events to `172.21.53.{1..3}` +which are internally exposed IP-s of the secondaries. + +# Secrets To configure TSIG secrets: @@ -61,7 +68,7 @@ zone "foobar.com" { file "/var/lib/bind/db.foobar.com"; allow-update { !rejected; key foobar; }; allow-transfer { !rejected; key readonly; key foobar; }; - notify explicit; also-notify { 172.20.53.1; 172.20.53.2; 172.20.53.3; }; + notify explicit; also-notify { 172.21.53.1; 172.21.53.2; 172.21.53.3; }; }; ``` diff --git a/bind/bind-secondary.yaml b/bind/bind-secondary.yaml index 4538f75..249e838 100644 --- a/bind/bind-secondary.yaml +++ b/bind/bind-secondary.yaml @@ -116,7 +116,7 @@ metadata: spec: type: LoadBalancer externalTrafficPolicy: Local - loadBalancerIP: 172.20.53.1 + loadBalancerIP: 172.21.53.1 selector: app: bind-secondary statefulset.kubernetes.io/pod-name: bind-secondary-0 @@ -138,7 +138,7 @@ metadata: spec: type: LoadBalancer externalTrafficPolicy: Local - loadBalancerIP: 172.20.53.2 + loadBalancerIP: 172.21.53.2 selector: app: bind-secondary statefulset.kubernetes.io/pod-name: bind-secondary-1 @@ -160,7 +160,7 @@ metadata: spec: type: LoadBalancer externalTrafficPolicy: Local - loadBalancerIP: 172.20.53.3 + loadBalancerIP: 172.21.53.3 selector: app: bind-secondary statefulset.kubernetes.io/pod-name: bind-secondary-2 diff --git a/elastic-system/application.yml b/elastic-system/application.yml index 50a285e..bd2fe04 100644 --- a/elastic-system/application.yml +++ b/elastic-system/application.yml @@ -150,7 +150,7 @@ metadata: spec: type: LoadBalancer externalTrafficPolicy: Local - loadBalancerIP: 172.20.51.4 + loadBalancerIP: 172.21.51.4 ports: - name: filebeat-syslog port: 514 @@ -169,7 +169,7 @@ metadata: spec: type: LoadBalancer externalTrafficPolicy: Local - loadBalancerIP: 172.20.51.4 + loadBalancerIP: 172.21.51.4 ports: - name: filebeat-syslog port: 514 diff --git a/metallb-system/application.yml b/metallb-system/application.yml index d44ba80..16386fc 100644 --- a/metallb-system/application.yml +++ b/metallb-system/application.yml @@ -26,19 +26,7 @@ metadata: namespace: metallb-system spec: addresses: - - 172.20.51.0/24 ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: zoo - namespace: metallb-system -spec: - ipAddressPools: - - zoo - - bind-secondary-external - - bind-secondary-internal - - wildduck + - 172.21.51.0/24 --- # Slice of public EEnet subnet using MetalLB L3 method apiVersion: metallb.io/v1beta1 @@ -67,7 +55,7 @@ metadata: namespace: metallb-system spec: addresses: - - 172.20.53.0/24 + - 172.21.53.0/24 --- apiVersion: metallb.io/v1beta1 kind: IPAddressPool @@ -99,13 +87,3 @@ spec: passwordSecret: name: mikrotik-router namespace: metallb-system ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: public - namespace: metallb-system -spec: - ipAddressPools: - - eenet - - elisa diff --git a/monitoring/blackbox-exporter.yaml b/monitoring/blackbox-exporter.yaml index 326c45e..11c800e 100644 --- a/monitoring/blackbox-exporter.yaml +++ b/monitoring/blackbox-exporter.yaml @@ -33,9 +33,9 @@ spec: static: - 193.40.103.2 - 62.65.250.2 - - 172.20.53.1 - - 172.20.53.2 - - 172.20.53.3 + - 172.21.53.1 + - 172.21.53.2 + - 172.21.53.3 --- apiVersion: monitoring.coreos.com/v1 kind: Probe diff --git a/tigera-operator/README.md b/tigera-operator/README.md index 8609429..82a455c 100644 --- a/tigera-operator/README.md +++ b/tigera-operator/README.md @@ -2,7 +2,7 @@ Calico implements [container network interface plugin](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) -which enables pods to talk to eachother. +which enables inter-pod network with BGP, also advertising Service LB IPs. # For user @@ -13,7 +13,8 @@ Nothing specific to point out, this is standard Kubernetes feature Tigera operator was used to deploy Calico: ``` -curl https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/tigera-operator.yaml -O +curl https://raw.githubusercontent.com/projectcalico/calico/v3.28.1/manifests/tigera-operator.yaml -O kubectl apply --server-side --force-conflicts -f tigera-operator.yaml kubectl apply -f application.yaml +kubectl -n calico-system create secret generic bgp-secrets --from-literal=password=... ``` diff --git a/tigera-operator/application.yaml b/tigera-operator/application.yaml index b06e711..37f4a06 100644 --- a/tigera-operator/application.yaml +++ b/tigera-operator/application.yaml @@ -12,8 +12,8 @@ spec: - blockSize: 26 cidr: 10.244.0.0/16 encapsulation: VXLANCrossSubnet - natOutgoing: Enabled - nodeSelector: all() + natOutgoing: Disabled + nodeSelector: all() --- # This section configures the Calico API server. # For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer @@ -22,3 +22,60 @@ kind: APIServer metadata: name: default spec: {} +--- +apiVersion: crd.projectcalico.org/v1 +kind: BGPPeer +metadata: + name: mikrotik-router +spec: + peerIP: 172.21.255.254 + asNumber: 64567 + keepOriginalNextHop: true + password: + secretKeyRef: + name: bgp-secrets + key: password +--- +apiVersion: crd.projectcalico.org/v1 +kind: BGPConfiguration +metadata: + name: default +spec: + serviceLoadBalancerIPs: + - cidr: 172.21.51.4/32 + - cidr: 172.21.53.1/32 + - cidr: 172.21.53.2/32 + - cidr: 172.21.53.3/32 + - cidr: 193.40.103.36/32 + - cidr: 193.40.103.37/32 + - cidr: 193.40.103.38/32 + - cidr: 193.40.103.39/32 + - cidr: 62.65.250.36/32 + - cidr: 62.65.250.37/32 + - cidr: 62.65.250.2/32 + - cidr: 193.40.103.25/32 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: secret-access + namespace: calico-system +rules: +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["bgp-secrets"] + verbs: ["watch", "list", "get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: secret-access + namespace: calico-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: secret-access +subjects: +- kind: ServiceAccount + name: calico-node + namespace: calico-system \ No newline at end of file