1
0
forked from k-space/kube

test proxmox csi

This commit is contained in:
Erki Aas 2024-08-23 17:10:42 +03:00
parent 67fb6c3727
commit c6c94b1901
4 changed files with 591 additions and 0 deletions

6
proxmox-csi/config.yaml Normal file
View File

@ -0,0 +1,6 @@
clusters:
- url: https://pve1.proxmox.infra.k-space.ee:8006/api2/json
insecure: false
token_id: "ks-kubernetes-csi@pve!cs"
token_secret: "<token goes here>"
region: pve-cluster

View File

@ -0,0 +1,552 @@
---
# Source: proxmox-csi-plugin/templates/namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: csi-proxmox
labels:
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/audit: baseline
pod-security.kubernetes.io/warn: baseline
---
# Source: proxmox-csi-plugin/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: proxmox-csi-plugin-controller
namespace: csi-proxmox
labels:
helm.sh/chart: proxmox-csi-plugin-0.2.8
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/version: "v0.7.0"
app.kubernetes.io/managed-by: Helm
---
# Source: proxmox-csi-plugin/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: proxmox-csi-plugin-node
namespace: csi-proxmox
labels:
helm.sh/chart: proxmox-csi-plugin-0.2.8
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/version: "v0.7.0"
app.kubernetes.io/managed-by: Helm
---
# Source: proxmox-csi-plugin/templates/storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: proxmox-data-xfs
provisioner: csi.proxmox.sinextra.dev
allowVolumeExpansion: true
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Delete
parameters:
csi.storage.k8s.io/fstype: xfs
storage: data
---
# Source: proxmox-csi-plugin/templates/storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: proxmox-data
provisioner: csi.proxmox.sinextra.dev
allowVolumeExpansion: true
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Delete
parameters:
csi.storage.k8s.io/fstype: ext4
ssd: "true"
storage: data
---
# Source: proxmox-csi-plugin/templates/controller-clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: proxmox-csi-plugin-controller
namespace: csi-proxmox
labels:
helm.sh/chart: proxmox-csi-plugin-0.2.8
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/version: "v0.7.0"
app.kubernetes.io/managed-by: Helm
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "patch", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
verbs: ["patch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get","list", "watch", "create", "update", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments/status"]
verbs: ["patch"]
---
# Source: proxmox-csi-plugin/templates/node-clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: proxmox-csi-plugin-node
namespace: csi-proxmox
labels:
helm.sh/chart: proxmox-csi-plugin-0.2.8
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/version: "v0.7.0"
app.kubernetes.io/managed-by: Helm
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
---
# Source: proxmox-csi-plugin/templates/controller-rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: proxmox-csi-plugin-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: proxmox-csi-plugin-controller
subjects:
- kind: ServiceAccount
name: proxmox-csi-plugin-controller
namespace: csi-proxmox
---
# Source: proxmox-csi-plugin/templates/node-rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: proxmox-csi-plugin-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: proxmox-csi-plugin-node
subjects:
- kind: ServiceAccount
name: proxmox-csi-plugin-node
namespace: csi-proxmox
---
# Source: proxmox-csi-plugin/templates/controller-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: proxmox-csi-plugin-controller
namespace: csi-proxmox
labels:
helm.sh/chart: proxmox-csi-plugin-0.2.8
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/version: "v0.7.0"
app.kubernetes.io/managed-by: Helm
rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: ["storage.k8s.io"]
resources: ["csistoragecapacities"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get"]
- apiGroups: ["apps"]
resources: ["replicasets"]
verbs: ["get"]
---
# Source: proxmox-csi-plugin/templates/controller-rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: proxmox-csi-plugin-controller
namespace: csi-proxmox
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: proxmox-csi-plugin-controller
subjects:
- kind: ServiceAccount
name: proxmox-csi-plugin-controller
namespace: csi-proxmox
---
# Source: proxmox-csi-plugin/templates/node-deployment.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: proxmox-csi-plugin-node
namespace: csi-proxmox
labels:
helm.sh/chart: proxmox-csi-plugin-0.2.8
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/version: "v0.7.0"
app.kubernetes.io/managed-by: Helm
spec:
updateStrategy:
type: RollingUpdate
selector:
matchLabels:
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/component: node
template:
metadata:
labels:
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/component: node
spec:
priorityClassName: system-node-critical
enableServiceLinks: false
serviceAccountName: proxmox-csi-plugin-node
securityContext:
runAsUser: 0
runAsGroup: 0
containers:
- name: proxmox-csi-plugin-node
securityContext:
privileged: true
capabilities:
drop:
- ALL
add:
- SYS_ADMIN
- CHOWN
- DAC_OVERRIDE
seccompProfile:
type: RuntimeDefault
image: "ghcr.io/sergelogvinov/proxmox-csi-node:edge"
imagePullPolicy: Always
args:
- "-v=5"
- "--csi-address=unix:///csi/csi.sock"
- "--node-id=$(NODE_NAME)"
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
resources:
{}
volumeMounts:
- name: socket
mountPath: /csi
- name: kubelet
mountPath: /var/lib/kubelet
mountPropagation: Bidirectional
- name: dev
mountPath: /dev
- name: sys
mountPath: /sys
- name: csi-node-driver-registrar
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
seccompProfile:
type: RuntimeDefault
image: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.4"
imagePullPolicy: IfNotPresent
args:
- "-v=5"
- "--csi-address=unix:///csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.proxmox.sinextra.dev/csi.sock"
volumeMounts:
- name: socket
mountPath: /csi
- name: registration
mountPath: /registration
resources:
requests:
cpu: 10m
memory: 16Mi
- name: liveness-probe
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
seccompProfile:
type: RuntimeDefault
image: "registry.k8s.io/sig-storage/livenessprobe:v2.11.0"
imagePullPolicy: IfNotPresent
args:
- "-v=5"
- "--csi-address=unix:///csi/csi.sock"
volumeMounts:
- name: socket
mountPath: /csi
resources:
requests:
cpu: 10m
memory: 16Mi
volumes:
- name: socket
hostPath:
path: /var/lib/kubelet/plugins/csi.proxmox.sinextra.dev/
type: DirectoryOrCreate
- name: registration
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: kubelet
hostPath:
path: /var/lib/kubelet
type: Directory
- name: dev
hostPath:
path: /dev
type: Directory
- name: sys
hostPath:
path: /sys
type: Directory
tolerations:
- effect: NoSchedule
key: node.kubernetes.io/unschedulable
operator: Exists
- effect: NoSchedule
key: node.kubernetes.io/disk-pressure
operator: Exists
---
# Source: proxmox-csi-plugin/templates/controller-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: proxmox-csi-plugin-controller
namespace: csi-proxmox
labels:
helm.sh/chart: proxmox-csi-plugin-0.2.8
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/version: "v0.7.0"
app.kubernetes.io/managed-by: Helm
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/component: controller
template:
metadata:
annotations:
checksum/config: c69436cb1e16c36ff708b1003d3ca4c6ee6484d2524e2ba7d9b68f473acaa1ca
labels:
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/component: controller
spec:
priorityClassName: system-cluster-critical
enableServiceLinks: false
serviceAccountName: proxmox-csi-plugin-controller
securityContext:
fsGroup: 65532
fsGroupChangePolicy: OnRootMismatch
runAsGroup: 65532
runAsNonRoot: true
runAsUser: 65532
hostAliases:
[]
initContainers:
[]
containers:
- name: proxmox-csi-plugin-controller
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
seccompProfile:
type: RuntimeDefault
image: "ghcr.io/sergelogvinov/proxmox-csi-controller:edge"
imagePullPolicy: Always
args:
- "-v=5"
- "--csi-address=unix:///csi/csi.sock"
- "--cloud-config=/etc/proxmox/config.yaml"
# - "--metrics-address=:8080"
# ports:
# - name: metrics
# containerPort: 8080
# protocol: TCP
resources:
requests:
cpu: 10m
memory: 16Mi
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: cloud-config
mountPath: /etc/proxmox/
- name: csi-attacher
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
seccompProfile:
type: RuntimeDefault
image: "registry.k8s.io/sig-storage/csi-attacher:v4.4.4"
imagePullPolicy: IfNotPresent
args:
- "-v=5"
- "--csi-address=unix:///csi/csi.sock"
- "--timeout=3m"
- "--leader-election"
- "--default-fstype=ext4"
volumeMounts:
- name: socket-dir
mountPath: /csi
resources:
requests:
cpu: 10m
memory: 16Mi
- name: csi-provisioner
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
seccompProfile:
type: RuntimeDefault
image: "registry.k8s.io/sig-storage/csi-provisioner:v3.6.4"
imagePullPolicy: IfNotPresent
args:
- "-v=5"
- "--csi-address=unix:///csi/csi.sock"
- "--timeout=3m"
- "--leader-election"
- "--default-fstype=ext4"
- "--feature-gates=Topology=True"
- "--enable-capacity"
- "--capacity-ownerref-level=2"
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
volumeMounts:
- name: socket-dir
mountPath: /csi
resources:
requests:
cpu: 10m
memory: 16Mi
- name: csi-resizer
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
seccompProfile:
type: RuntimeDefault
image: "registry.k8s.io/sig-storage/csi-resizer:v1.9.4"
imagePullPolicy: IfNotPresent
args:
- "-v=5"
- "--csi-address=unix:///csi/csi.sock"
- "--timeout=3m"
- "--handle-volume-inuse-error=false"
- "--leader-election"
volumeMounts:
- name: socket-dir
mountPath: /csi
resources:
requests:
cpu: 10m
memory: 16Mi
- name: liveness-probe
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
seccompProfile:
type: RuntimeDefault
image: "registry.k8s.io/sig-storage/livenessprobe:v2.11.0"
imagePullPolicy: IfNotPresent
args:
- "-v=5"
- "--csi-address=unix:///csi/csi.sock"
volumeMounts:
- name: socket-dir
mountPath: /csi
resources:
requests:
cpu: 10m
memory: 16Mi
volumes:
- name: socket-dir
emptyDir: {}
- name: cloud-config
secret:
secretName: proxmox-csi-plugin
nodeSelector:
node-role.kubernetes.io/control-plane: ""
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: proxmox-csi-plugin
app.kubernetes.io/instance: proxmox-csi-plugin
app.kubernetes.io/component: controller
---
# Source: proxmox-csi-plugin/templates/csidriver.yaml
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: csi.proxmox.sinextra.dev
spec:
attachRequired: true
podInfoOnMount: true
storageCapacity: true
volumeLifecycleModes:
- Persistent

19
proxmox-csi/readme.md Normal file
View File

@ -0,0 +1,19 @@
# proxmox-csi
1. create role in pve if it doesn't exist
2. create user and assign permissions, preferrably at resource pool level
```
pveum user add ks-kubernetes-csi@pve
pveum aclmod /pool/kspace_pool -user ks-kubernetes-csi@pve -role CSI
pveum user token add ks-kubernetes-csi@pve cs -privsep 0
```
save the token!
3. apply `proxmox-csi-plugin.yml` and `storage-class.yaml`, delete proxmox-csi default storage classes from kube.
4. add the token from pve to `config.yaml` and create the secret: `kubectl -n csi-proxmox create secret generic proxmox-csi-plugin --from-file=config.yaml`
5. label the nodes according to allocation:
```
kubectl --kubeconfig ~/.kube/k-space label nodes worker1.kube.k-space.ee topology.kubernetes.io/region=pve-cluster topology.kubernetes.io/zone=pve1 --overwrite
kubectl --kubeconfig ~/.kube/k-space label nodes worker2.kube.k-space.ee topology.kubernetes.io/region=pve-cluster topology.kubernetes.io/zone=pve2 --overwrite
kubectl --kubeconfig ~/.kube/k-space label nodes worker3.kube.k-space.ee topology.kubernetes.io/region=pve-cluster topology.kubernetes.io/zone=pve8 --overwrite
kubectl --kubeconfig ~/.kube/k-space label nodes worker4.kube.k-space.ee topology.kubernetes.io/region=pve-cluster topology.kubernetes.io/zone=pve9 --overwrite
```

View File

@ -0,0 +1,14 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: proxmox
parameters:
csi.storage.k8s.io/fstype: xfs
storage: ks-pvs
cache: none
ssd: "false"
shared: "true"
provisioner: csi.proxmox.sinextra.dev
allowVolumeExpansion: true
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer