Initial commit

rosdump/.gitignore

@@ -0,0 +1,3 @@
+rosdump
+rosdump.pub
+ssh_known_hosts

rosdump/README.md

@@ -0,0 +1,68 @@
+# Intro
+
+This is how we make backups of Mikrotik device configurations using Kubernetes
+Cronjob. This is easy to monitor with Prometheus and integrates well with the
+rest of our montioring system. Also the script/manifest is less than 100 lines,
+easy to follow and to fix.
+
+Note that this does not have anything to do with
+[ecadlabs/rosdump](https://github.com/ecadlabs/rosdump)
+we initially used which just generated empty commits and
+there was no easy way to monitor.
+
+We also considered [ytti/oxidized](https://github.com/ytti/oxidized),
+but it does not export Prometheus metrics either.
+
+
+# Deployment
+
+To apply changes run in this directory:
+
+```
+kubectl apply -n rosdump -f cronjob.yaml
+```
+
+To trigger cronjob:
+
+```
+kubectl create job -n rosdump --from=cronjob/rosdump-cronjob rosdump-job-oneshot
+```
+
+For alerting:
+
+```
+absent(kube_cronjob_status_last_successful_time{cronjob="rosdump-cronjob"})
+```
+
+# Updating SSH public keys
+
+Whenever Mikrotik targets are added/removed or if their SSH keys change,
+use following to apply changes:
+
+```
+(for j in $(kubectl get cm -n rosdump rosdump-config -o json | jq -r '.data.targets');  do ssh-keyscan -t rsa $j; done) > ssh_known_hosts
+kubectl delete -n rosdump configmap rosdump-known-hosts
+kubectl create -n rosdump configmap rosdump-known-hosts --from-file=ssh_known_hosts
+```
+
+Make sure strong crypto is enabled on Mikrotik side:
+
+```
+/ip ssh set strong-crypto=yes allow-none-crypto=no
+```
+
+
+# Replacing SSH private key
+
+This affects access to both Gitea and Mikrotik targets.
+
+Generate new key and inject it to Kubernetes cluster:
+
+```
+rm -fv rosdump
+ssh-keygen -P '' -b 2048 -m PEM -t rsa -f rosdump -C rosdump
+kubectl delete -n rosdump secret rosdump-secrets
+kubectl create -n rosdump secret generic rosdump-secrets --from-file=ssh_identity=rosdump
+```
+
+Proceed to replace the public key in Gitea with one from `rosdump.pub`

rosdump/application.yml

@@ -0,0 +1,110 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rosdump-config
+data:
+  script.sh: |
+    #!/bin/bash
+    set -e
+    if [ -d rosdump ]; then
+        echo "Pulling Git repo"
+        cd rosdump
+        git pull
+    else
+        echo "Cloning Git repo"
+        git clone git@git.k-space.ee:k-space/rosdump.git
+        cd rosdump
+    fi
+    git rm *.k-space.ee
+    for target in $(cat /config/targets | grep -v '^#'); do
+        echo "Exporting configuration for $target"
+        ssh rosdump@$target '/export' | grep -v '^# serial number =' | grep -v '^#.* by RouterOS' > $target
+        git add $target
+    done
+    if [[ `git status --porcelain` ]]; then  
+      echo "Attempting Git check in"
+      git commit -m "Update $(git ls-files -m) file(s)"
+      git push
+    else
+      echo "No changes to commit"
+    fi
+  targets: |
+    router.mgmt.k-space.ee
+    sw_core01.mgmt.k-space.ee
+    sw_core02.mgmt.k-space.ee
+    sw_mgmt.mgmt.k-space.ee
+    sw_poe.mgmt.k-space.ee
+    sw_ha.mgmt.k-space.ee
+    sw_cyber.mgmt.k-space.ee
+    sw_chaos.mgmt.k-space.ee
+    sw_asocial.mgmt.k-space.ee
+    sw_kitchen.mgmt.k-space.ee
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: rosdump-cronjob
+spec:
+  schedule: "0 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 300
+      template:
+        spec:
+          nodeSelector:
+            dedicated: monitoring
+          tolerations:
+          - key: dedicated
+            operator: Equal
+            value: monitoring
+            effect: NoSchedule
+          restartPolicy: OnFailure
+          containers:
+          - name: rosdump
+            image: harbor.k-space.ee/k-space/microscript-base
+            imagePullPolicy: Always
+            args:
+              - bash
+              - /config/script.sh
+            volumeMounts:
+            - name: config
+              mountPath: /config
+          volumes:
+            - name: config
+              projected:
+                sources:
+                  - secret:
+                      name: rosdump-secrets
+                      items:
+                        - key: ssh_identity
+                          path: ssh_identity
+                          mode: 0600
+                  - configMap:
+                      name: rosdump-known-hosts
+                      items:
+                        - key: ssh_known_hosts
+                          path: ssh_known_hosts
+                  - configMap:
+                      name: rosdump-config
+
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: rosdump
+spec:
+  podSelector: {}
+  policyTypes:
+  - Egress
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 193.40.103.0/24
+    - ipBlock:
+        cidr: 172.23.0.0/24
+    - ipBlock:
+        cidr: 100.102.1.0/24
+    ports:
+    - protocol: TCP
+      port: 22