hackerspace WIP

CLUSTER.md

@@ -61,24 +61,44 @@ Network → VPN → `IPv6` → Search domains (Otsingudomeenid): `kube.k-space.e
 Created Ubuntu 22.04 VM-s on Proxmox with local storage.
 Added some ARM64 workers by using Ubuntu 22.04 server on Raspberry Pi.
 
-First master:
+After machines have booted up and you can reach them via SSH:
+
+```
+# Disable Ubuntu caching DNS resolver
+systemctl disable systemd-resolved.service
+systemctl stop systemd-resolved
+rm -fv /etc/resolv.conf
+cat > /etc/resolv.conf << EOF
+nameserver 1.1.1.1
+nameserver 8.8.8.8
+EOF
+
+# Disable multipathd as Longhorn handles that itself
+systemctl mask multipathd snapd
+systemctl disable --now multipathd snapd bluetooth ModemManager hciuart wpa_supplicant packagekit
+
+# Permit root login
+sed -i -e 's/PermitRootLogin no/PermitRootLogin without-password/' /etc/ssh/sshd_config
+systemctl reload ssh
+cat ~ubuntu/.ssh/authorized_keys > /root/.ssh/authorized_keys
+userdel -f ubuntu
+apt-get install -yqq linux-image-generic
+apt-get remove -yq cloud-init linux-image-*-kvm
+```
+
+On master:
+
 ```
 kubeadm init --token-ttl=120m --pod-network-cidr=10.244.0.0/16 --control-plane-endpoint "master.kube.k-space.ee:6443" --upload-certs --apiserver-cert-extra-sans master.kube.k-space.ee --node-name master1.kube.k-space.ee
 ```
 
-Joining nodes:
+For the `kubeadm join` command specify FQDN via `--node-name $(hostname -f)`.
-```
-# On a master:
-kubeadm token create --print-join-command
-
-# Joining node:
-<printed join command --node-name "$(hostname -f)"
-```
 
 Set AZ labels:
+
 ```
 for j in $(seq 1 9); do
-  for t in master mon worker; do
+  for t in master mon worker storage; do
     kubectl label nodes ${t}${j}.kube.k-space.ee topology.kubernetes.io/zone=node${j}
   done
 done
@@ -95,6 +115,11 @@ for j in $(seq 1 4); do
   kubectl taint nodes mon${j}.kube.k-space.ee dedicated=monitoring:NoSchedule
   kubectl label nodes mon${j}.kube.k-space.ee dedicated=monitoring
 done
+
+for j in $(seq 1 4); do
+  kubectl taint nodes storage${j}.kube.k-space.ee dedicated=storage:NoSchedule
+  kubectl label nodes storage${j}.kube.k-space.ee dedicated=storage
+done
 ```
 
 For `arm64` nodes add suitable taint to prevent scheduling non-multiarch images on them:
@@ -112,6 +137,13 @@ for j in ground front back; do
 done
 ```
 
+To reduce wear on storage:
+```
+echo StandardOutput=null >> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+systemctl daemon-reload
+systemctl restart kubelet
+```
+
 ## Technology mapping
 Our self-hosted Kubernetes stack compared to AWS based deployments:
 

hackerspace/README.md

@@ -1,20 +1,8 @@
-## hackerspace / inventory
+## inventory.k-space.ee
+Reads-writes to mongo.
+
 <!-- Referenced/linked by https://wiki.k-space.ee/en/hosting/doors -->
+A component of inventory is 'doorboy' (https://wiki.k-space.ee/en/hosting/doors)
 
-## [doorboy-proxy](https://github.com/k-space/doorboy-proxy)
+## k6.ee
-- Dispatches open events (from mongodb) to door controllers.
-- Handles Slack open events (to mongodb).
-- Forwards logs from door controllers to mongodb.
-- Broadcasts mongodb logs to Slack.
-
-See also:
-- inventory-app door components
-- https://wiki.k-space.ee/en/hosting/doors
-
-## [inventory-app](https://github.com/k-space/inventory-app) (inventory.k-space.ee)
-- Inventory
-- Manages door keycards.
-- Forwards door opens from website to mongodb (what are picked up by doorboy-proxy).
-
-## [goredirect](https://github.com/k-space/goredirect) (k6.ee)
 Reads from mongo, HTTP redirect to //inventory.k-space.ee/m/inventory/{uuid}/view

hackerspace/doorboy.yaml

@@ -26,7 +26,6 @@ spec:
                       - doorboy-proxy
                 topologyKey: topology.kubernetes.io/zone
               weight: 100
-      serviceAccountName: inventory-svcacc
       containers:
         - name: doorboy-proxy
           image: harbor.k-space.ee/k-space/doorboy-proxy:latest
@@ -34,14 +33,16 @@ spec:
             - secretRef:
                 name: inventory-mongodb
             - secretRef:
-                name: doorboy-godoor
+                name: doorboy-api
-            - secretRef:
-                name: doorboy-slack
           env:
             - name: OIDC_USERS_NAMESPACE
               value: passmower
             - name: SLACK_CHANNEL_ID
               value: CDL9H8Q9W
+            - name: FLOOR_ACCESS_GROUP
+              value: 'k-space:floor'
+            - name: WORKSHOP_ACCESS_GROUP
+              value: 'k-space:workshop'
           securityContext:
             readOnlyRootFilesystem: true
             runAsNonRoot: true

hackerspace/inventory-extras.yaml

@@ -1,24 +1,37 @@
----
+apiVersion: traefik.io/v1alpha1
-apiVersion: codemowers.cloud/v1beta1
+kind: Middleware
-kind: OIDCClient
 metadata:
-  name: inventory-app
+  name: members-inventory-redirect
 spec:
-  uri: 'https://inventory.k-space.ee'
+  redirectRegex:
-  redirectUris:
+    regex: ^https://members.k-space.ee/(.*)
-    - 'https://inventory.k-space.ee/login-callback'
+    replacement: https://inventory.k-space.ee/${1}
-  grantTypes:
+    permanent: false
-    - 'authorization_code'
+---
-    - 'refresh_token'
+# Creates a dummy/stub in auth.k-space.ee user-facing service listing (otherwise only inventory.k-space.ee is listed).
-  responseTypes:
+apiVersion: codemowers.cloud/v1beta1
-    - 'code'
+kind: OIDCMiddlewareClient
-  availableScopes:
+metadata:
-    - 'openid'
+  name: doorboy
-    - 'profile'
+spec:
-    - 'groups'
+  displayName: Doorboy
-    - 'offline_access'
+  uri: 'https://inventory.k-space.ee/m/doorboy'
-  tokenEndpointAuthMethod: 'client_secret_basic'
+---
-  pkce: false
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: members-inventory
+spec:
+  entryPoints:
+    - websecure
+  routes:
+  - match: Host(`members.k-space.ee`)
+    kind: Rule
+    middlewares:
+      - name: members-inventory-redirect
+    services:
+      - kind: TraefikService
+        name: api@internal
 ---
 apiVersion: codemowers.cloud/v1beta1
 kind: MinioBucketClaim

hackerspace/inventory-redirects.yaml

@@ -1,35 +0,0 @@
----
-# Creates a dummy/stub in auth.k-space.ee user-facing service listing (otherwise only inventory.k-space.ee is listed).
-apiVersion: codemowers.cloud/v1beta1
-kind: OIDCMiddlewareClient
-metadata:
-  name: doorboy
-spec:
-  displayName: Doorboy
-  uri: 'https://inventory.k-space.ee/m/doorboy'
----
-apiVersion: traefik.io/v1alpha1
-kind: Middleware
-metadata:
-  name: members-inventory-redirect
-spec:
-  redirectRegex:
-    regex: ^https://members.k-space.ee/(.*)
-    replacement: https://inventory.k-space.ee/${1}
-    permanent: false
----
-apiVersion: traefik.io/v1alpha1
-kind: IngressRoute
-metadata:
-  name: members-inventory
-spec:
-  entryPoints:
-    - websecure
-  routes:
-  - match: Host(`members.k-space.ee`)
-    kind: Rule
-    middlewares:
-      - name: members-inventory-redirect
-    services:
-      - kind: TraefikService
-        name: api@internal

hackerspace/inventory.yaml

@@ -20,12 +20,31 @@ spec:
       - image: harbor.k-space.ee/k-space/inventory-app:latest
         imagePullPolicy: Always
         env:
+        - name: ENVIRONMENT_TYPE
+          value: PROD
+        - name: PYTHONUNBUFFERED
+          value: "1"
         - name: INVENTORY_ASSETS_BASE_URL
           value: https://external.minio-clusters.k-space.ee/hackerspace-701d9303-0f27-4829-a2be-b1084021ad91/
         - name: MACADDRESS_OUTLINK_BASEURL
           value: https://grafana.k-space.ee/d/ddwyidbtbc16oa/ip-usage?orgId=1&from=now-2y&to=now&timezone=browser&var-Filters=mac%7C%3D%7C
         - name: OIDC_USERS_NAMESPACE
           value: passmower
+        - name: SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              key: SECRET_KEY
+              name: inventory-secrets
+        - name: SLACK_DOORLOG_CALLBACK
+          valueFrom:
+            secretKeyRef:
+              key: SLACK_DOORLOG_CALLBACK
+              name: slack-secrets
+        - name: SLACK_VERIFICATION_TOKEN
+          valueFrom:
+            secretKeyRef:
+              key: SLACK_VERIFICATION_TOKEN
+              name: slack-secrets
         envFrom:
         - secretRef:
             name: miniobucket-inventory-external-owner-secrets
@@ -98,3 +117,59 @@ spec:
   tls:
   - hosts:
     - "*.k-space.ee"
+---
+apiVersion: codemowers.cloud/v1beta1
+kind: OIDCClient
+metadata:
+  name: inventory-app
+spec:
+  uri: 'https://inventory.k-space.ee'
+  redirectUris:
+    - 'https://inventory.k-space.ee/login-callback'
+  grantTypes:
+    - 'authorization_code'
+    - 'refresh_token'
+  responseTypes:
+    - 'code'
+  availableScopes:
+    - 'openid'
+    - 'profile'
+    - 'groups'
+    - 'offline_access'
+  tokenEndpointAuthMethod: 'client_secret_basic'
+  pkce: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: inventory-role
+  namespace: hackerspace
+rules:
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - codemowers.cloud
+    resources:
+      - oidcusers
+      - oidcusers/status
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: inventory-roles
+  namespace: hackerspace
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: inventory-role
+subjects:
+  - kind: ServiceAccount
+    name: inventory-svcacc
+    namespace: hackerspace
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: inventory-svcacc

hackerspace/kustomization.yaml

@@ -4,10 +4,9 @@ kind: Kustomization
 namespace: hackerspace
 
 resources:
-- ssh://git@git.k-space.ee/secretspace/kube/hackerspace # secrets: inventory-mongodb, inventory-s3, doorboy-godoor, doorboy-slack
+- ssh://git@git.k-space.ee/secretspace/kube/hackerspace # secret: grafana-database
 - ./doorboy.yaml
-- ./svcacc.yaml
+- ./goredirect.yaml
 - ./inventory.yaml
 - ./inventory-extras.yaml
-- ./inventory-redirects.yaml
+- https://github.com/mongodb/mongodb-kubernetes-operator//config/rbac/?ref=v0.13.0
-- ./goredirect.yaml

hackerspace/svcacc.yaml

@@ -1,35 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: inventory-role
-  namespace: hackerspace
-rules:
-  - verbs:
-      - get
-      - list
-      - watch
-    apiGroups:
-      - codemowers.cloud
-    resources:
-      - oidcusers
-      - oidcusers/status
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: inventory-roles
-  namespace: hackerspace
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: inventory-role
-subjects:
-  - kind: ServiceAccount
-    name: inventory-svcacc
-    namespace: hackerspace
----
-# used by inventory and doorboy
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: inventory-svcacc