From c6c94b1901867c0c232d10125fd6437f945d3474 Mon Sep 17 00:00:00 2001 From: Erki Aas Date: Fri, 23 Aug 2024 17:10:42 +0300 Subject: [PATCH] test proxmox csi --- proxmox-csi/config.yaml | 6 + proxmox-csi/proxmox-csi-plugin.yml | 552 +++++++++++++++++++++++++++++ proxmox-csi/readme.md | 19 + proxmox-csi/storage-class.yaml | 14 + 4 files changed, 591 insertions(+) create mode 100644 proxmox-csi/config.yaml create mode 100644 proxmox-csi/proxmox-csi-plugin.yml create mode 100644 proxmox-csi/readme.md create mode 100644 proxmox-csi/storage-class.yaml diff --git a/proxmox-csi/config.yaml b/proxmox-csi/config.yaml new file mode 100644 index 0000000..66141ac --- /dev/null +++ b/proxmox-csi/config.yaml @@ -0,0 +1,6 @@ +clusters: + - url: https://pve1.proxmox.infra.k-space.ee:8006/api2/json + insecure: false + token_id: "ks-kubernetes-csi@pve!cs" + token_secret: "" + region: pve-cluster \ No newline at end of file diff --git a/proxmox-csi/proxmox-csi-plugin.yml b/proxmox-csi/proxmox-csi-plugin.yml new file mode 100644 index 0000000..75eba92 --- /dev/null +++ b/proxmox-csi/proxmox-csi-plugin.yml @@ -0,0 +1,552 @@ +--- +# Source: proxmox-csi-plugin/templates/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: csi-proxmox + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: baseline + pod-security.kubernetes.io/warn: baseline +--- +# Source: proxmox-csi-plugin/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-csi-plugin-controller + namespace: csi-proxmox + labels: + helm.sh/chart: proxmox-csi-plugin-0.2.8 + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/version: "v0.7.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: proxmox-csi-plugin/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-csi-plugin-node + namespace: csi-proxmox + labels: + helm.sh/chart: proxmox-csi-plugin-0.2.8 + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/version: "v0.7.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: proxmox-csi-plugin/templates/storageclass.yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: proxmox-data-xfs +provisioner: csi.proxmox.sinextra.dev +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +parameters: + csi.storage.k8s.io/fstype: xfs + storage: data +--- +# Source: proxmox-csi-plugin/templates/storageclass.yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: proxmox-data +provisioner: csi.proxmox.sinextra.dev +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +parameters: + csi.storage.k8s.io/fstype: ext4 + ssd: "true" + storage: data +--- +# Source: proxmox-csi-plugin/templates/controller-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxmox-csi-plugin-controller + namespace: csi-proxmox + labels: + helm.sh/chart: proxmox-csi-plugin-0.2.8 + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/version: "v0.7.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get","list", "watch", "create", "update", "patch"] + + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +--- +# Source: proxmox-csi-plugin/templates/node-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxmox-csi-plugin-node + namespace: csi-proxmox + labels: + helm.sh/chart: proxmox-csi-plugin-0.2.8 + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/version: "v0.7.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get +--- +# Source: proxmox-csi-plugin/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxmox-csi-plugin-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxmox-csi-plugin-controller +subjects: + - kind: ServiceAccount + name: proxmox-csi-plugin-controller + namespace: csi-proxmox +--- +# Source: proxmox-csi-plugin/templates/node-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxmox-csi-plugin-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxmox-csi-plugin-node +subjects: + - kind: ServiceAccount + name: proxmox-csi-plugin-node + namespace: csi-proxmox +--- +# Source: proxmox-csi-plugin/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: proxmox-csi-plugin-controller + namespace: csi-proxmox + labels: + helm.sh/chart: proxmox-csi-plugin-0.2.8 + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/version: "v0.7.0" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + + - apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] +--- +# Source: proxmox-csi-plugin/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: proxmox-csi-plugin-controller + namespace: csi-proxmox +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: proxmox-csi-plugin-controller +subjects: + - kind: ServiceAccount + name: proxmox-csi-plugin-controller + namespace: csi-proxmox +--- +# Source: proxmox-csi-plugin/templates/node-deployment.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: proxmox-csi-plugin-node + namespace: csi-proxmox + labels: + helm.sh/chart: proxmox-csi-plugin-0.2.8 + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/version: "v0.7.0" + app.kubernetes.io/managed-by: Helm +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/component: node + template: + metadata: + labels: + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/component: node + spec: + priorityClassName: system-node-critical + enableServiceLinks: false + serviceAccountName: proxmox-csi-plugin-node + securityContext: + runAsUser: 0 + runAsGroup: 0 + containers: + - name: proxmox-csi-plugin-node + securityContext: + privileged: true + capabilities: + drop: + - ALL + add: + - SYS_ADMIN + - CHOWN + - DAC_OVERRIDE + seccompProfile: + type: RuntimeDefault + image: "ghcr.io/sergelogvinov/proxmox-csi-node:edge" + imagePullPolicy: Always + args: + - "-v=5" + - "--csi-address=unix:///csi/csi.sock" + - "--node-id=$(NODE_NAME)" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + {} + volumeMounts: + - name: socket + mountPath: /csi + - name: kubelet + mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + - name: dev + mountPath: /dev + - name: sys + mountPath: /sys + - name: csi-node-driver-registrar + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.4" + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--csi-address=unix:///csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.proxmox.sinextra.dev/csi.sock" + volumeMounts: + - name: socket + mountPath: /csi + - name: registration + mountPath: /registration + resources: + requests: + cpu: 10m + memory: 16Mi + - name: liveness-probe + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "registry.k8s.io/sig-storage/livenessprobe:v2.11.0" + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--csi-address=unix:///csi/csi.sock" + volumeMounts: + - name: socket + mountPath: /csi + resources: + requests: + cpu: 10m + memory: 16Mi + volumes: + - name: socket + hostPath: + path: /var/lib/kubelet/plugins/csi.proxmox.sinextra.dev/ + type: DirectoryOrCreate + - name: registration + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet + hostPath: + path: /var/lib/kubelet + type: Directory + - name: dev + hostPath: + path: /dev + type: Directory + - name: sys + hostPath: + path: /sys + type: Directory + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/unschedulable + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/disk-pressure + operator: Exists +--- +# Source: proxmox-csi-plugin/templates/controller-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: proxmox-csi-plugin-controller + namespace: csi-proxmox + labels: + helm.sh/chart: proxmox-csi-plugin-0.2.8 + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/version: "v0.7.0" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/component: controller + template: + metadata: + annotations: + checksum/config: c69436cb1e16c36ff708b1003d3ca4c6ee6484d2524e2ba7d9b68f473acaa1ca + labels: + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/component: controller + spec: + priorityClassName: system-cluster-critical + enableServiceLinks: false + serviceAccountName: proxmox-csi-plugin-controller + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + hostAliases: + [] + initContainers: + [] + containers: + - name: proxmox-csi-plugin-controller + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "ghcr.io/sergelogvinov/proxmox-csi-controller:edge" + imagePullPolicy: Always + args: + - "-v=5" + - "--csi-address=unix:///csi/csi.sock" + - "--cloud-config=/etc/proxmox/config.yaml" + # - "--metrics-address=:8080" + # ports: + # - name: metrics + # containerPort: 8080 + # protocol: TCP + resources: + requests: + cpu: 10m + memory: 16Mi + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: cloud-config + mountPath: /etc/proxmox/ + - name: csi-attacher + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "registry.k8s.io/sig-storage/csi-attacher:v4.4.4" + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout=3m" + - "--leader-election" + - "--default-fstype=ext4" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + cpu: 10m + memory: 16Mi + - name: csi-provisioner + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "registry.k8s.io/sig-storage/csi-provisioner:v3.6.4" + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout=3m" + - "--leader-election" + - "--default-fstype=ext4" + - "--feature-gates=Topology=True" + - "--enable-capacity" + - "--capacity-ownerref-level=2" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + cpu: 10m + memory: 16Mi + - name: csi-resizer + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "registry.k8s.io/sig-storage/csi-resizer:v1.9.4" + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--csi-address=unix:///csi/csi.sock" + - "--timeout=3m" + - "--handle-volume-inuse-error=false" + - "--leader-election" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + cpu: 10m + memory: 16Mi + - name: liveness-probe + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + image: "registry.k8s.io/sig-storage/livenessprobe:v2.11.0" + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--csi-address=unix:///csi/csi.sock" + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + requests: + cpu: 10m + memory: 16Mi + volumes: + - name: socket-dir + emptyDir: {} + - name: cloud-config + secret: + secretName: proxmox-csi-plugin + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: proxmox-csi-plugin + app.kubernetes.io/instance: proxmox-csi-plugin + app.kubernetes.io/component: controller +--- +# Source: proxmox-csi-plugin/templates/csidriver.yaml +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.proxmox.sinextra.dev +spec: + attachRequired: true + podInfoOnMount: true + storageCapacity: true + volumeLifecycleModes: + - Persistent diff --git a/proxmox-csi/readme.md b/proxmox-csi/readme.md new file mode 100644 index 0000000..cc956ca --- /dev/null +++ b/proxmox-csi/readme.md @@ -0,0 +1,19 @@ +# proxmox-csi + +1. create role in pve if it doesn't exist +2. create user and assign permissions, preferrably at resource pool level + ``` + pveum user add ks-kubernetes-csi@pve + pveum aclmod /pool/kspace_pool -user ks-kubernetes-csi@pve -role CSI + pveum user token add ks-kubernetes-csi@pve cs -privsep 0 + ``` + save the token! +3. apply `proxmox-csi-plugin.yml` and `storage-class.yaml`, delete proxmox-csi default storage classes from kube. +4. add the token from pve to `config.yaml` and create the secret: `kubectl -n csi-proxmox create secret generic proxmox-csi-plugin --from-file=config.yaml` +5. label the nodes according to allocation: + ``` + kubectl --kubeconfig ~/.kube/k-space label nodes worker1.kube.k-space.ee topology.kubernetes.io/region=pve-cluster topology.kubernetes.io/zone=pve1 --overwrite + kubectl --kubeconfig ~/.kube/k-space label nodes worker2.kube.k-space.ee topology.kubernetes.io/region=pve-cluster topology.kubernetes.io/zone=pve2 --overwrite + kubectl --kubeconfig ~/.kube/k-space label nodes worker3.kube.k-space.ee topology.kubernetes.io/region=pve-cluster topology.kubernetes.io/zone=pve8 --overwrite + kubectl --kubeconfig ~/.kube/k-space label nodes worker4.kube.k-space.ee topology.kubernetes.io/region=pve-cluster topology.kubernetes.io/zone=pve9 --overwrite + ``` \ No newline at end of file diff --git a/proxmox-csi/storage-class.yaml b/proxmox-csi/storage-class.yaml new file mode 100644 index 0000000..828f3a1 --- /dev/null +++ b/proxmox-csi/storage-class.yaml @@ -0,0 +1,14 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: proxmox +parameters: + csi.storage.k8s.io/fstype: xfs + storage: ks-pvs + cache: none + ssd: "false" + shared: "true" +provisioner: csi.proxmox.sinextra.dev +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer