Support creating snapshots from btrfs volumes

Dockerfile

@@ -1,11 +1,18 @@
+FROM python:3.10-slim-buster as builder
+RUN apt-get update && apt-get install -y build-essential libbtrfsutil-dev
+RUN pip wheel -w /wheels "https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python"
+
 FROM python:3.10-slim-buster
 
 WORKDIR /app/
 
 RUN apt-get update && \
-    apt-get install -y e2fsprogs btrfs-progs xfsprogs && \
+    apt-get install -y e2fsprogs btrfs-progs libbtrfsutil1 xfsprogs && \
     rm -rf /var/lib/apt/lists/*
 
+COPY --from=builder /wheels/ /wheels/
+RUN pip install /wheels/*
+
 ENV PIP_NO_CACHE_DIR 1
 ADD ./requirements.txt ./
 RUN pip install -r ./requirements.txt

README.md

@@ -48,7 +48,7 @@ Features
 - [ ] Online shrinking: If fs supports it (e.g. btrfs)
 - [ ] Offline expansion/shrinking
 - [ ] Ephemeral inline volume
-- [ ] Snapshots: If the fs supports it (e.g. btrfs)
+- [x] Filesystem-level snapshots: `btrfs` supported
 
 Motivation
 ---

bd2fs.py

@@ -10,6 +10,8 @@ from csi.csi_pb2 import (
     NodeExpandVolumeRequest,
     CreateVolumeRequest,
 )
+from google.protobuf.timestamp_pb2 import Timestamp
+
 from declarative import (
     be_mounted,
     be_unmounted,
@@ -18,6 +20,8 @@ from declarative import (
     be_fs_expanded,
 )
 from fs_util import path_stats, mountpoint_to_dev
+from orchestrator.k8s import volume_to_node, run_on_node
+from remote import btrfs_create_snapshot, btrfs_delete_snapshot
 from util import log_grpc_request
 
 
@@ -218,3 +222,32 @@ class Bd2FsControllerServicer(csi_pb2_grpc.ControllerServicer):
         response = self.bds.ControllerExpandVolume(request, context)
         assert response.node_expansion_required
         return response
+
+    @log_grpc_request
+    def CreateSnapshot(self, request: csi_pb2.CreateSnapshotRequest, context):
+        volume_id = request.source_volume_id
+        name = request.name
+
+        snapshot_id, creation_time_ns = btrfs_create_snapshot(
+            volume_id=volume_id, name=name
+        )
+
+        nano = 10**9
+        return csi_pb2.CreateSnapshotResponse(
+            snapshot=csi_pb2.Snapshot(
+                size_bytes=0,
+                snapshot_id=snapshot_id,
+                source_volume_id=volume_id,
+                creation_time=Timestamp(
+                    seconds=creation_time_ns // nano, nanos=creation_time_ns % nano
+                ),
+                ready_to_use=True,
+            )
+        )
+
+    @log_grpc_request
+    def DeleteSnapshot(self, request: csi_pb2.DeleteSnapshotRequest, context):
+        snapshot_id = request.snapshot_id
+        volume_id, name = snapshot_id.rsplit("/", 1)
+        btrfs_delete_snapshot(volume_id=volume_id, name=name)
+        return csi_pb2.DeleteSnapshotResponse()

deploy/charts/rawfile-csi/templates/00-rbac.yaml

@@ -124,3 +124,34 @@ roleRef:
   kind: ClusterRole
   name: {{ include "rawfile-csi.fullname" . }}-resizer
   apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "rawfile-csi.fullname" . }}-snapshotter
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "rawfile-csi.fullname" . }}-snapshotter
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "rawfile-csi.fullname" . }}-driver
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "rawfile-csi.fullname" . }}-snapshotter
+  apiGroup: rbac.authorization.k8s.io

deploy/charts/rawfile-csi/templates/01-node-plugin.yaml

@@ -153,3 +153,20 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+        - name: external-snapshotter
+          image: gcr.io/k8s-staging-sig-storage/csi-snapshotter:v5.0.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--node-deployment=true"
+            - "--extra-create-metadata=true"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi

rawfile_servicer.py

@@ -141,6 +141,7 @@ class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer):
                 Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)),
                 Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)),
                 Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
+                Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_SNAPSHOT)),
             ]
         )
 

remote.py

@@ -1,3 +1,5 @@
+from contextlib import contextmanager
+
 from util import remote_fn
 
 
@@ -81,3 +83,56 @@ def expand_rawfile(volume_id, size):
         {"size": size},
     )
     run(f"truncate -s {size} {img_file}")
+
+
+@contextmanager
+def mount_root_subvol(volume_id):
+    import tempfile
+    import pathlib
+
+    import rawfile_util
+    from util import run
+
+    root_subvol = tempfile.mkdtemp(prefix="rawfile-")
+
+    img_file = rawfile_util.img_file(volume_id)
+    loop_dev = rawfile_util.attach_loop(img_file)
+
+    run(f"mount -t btrfs -o subvolid=0 {loop_dev} {root_subvol}")
+    try:
+        yield root_subvol
+    finally:
+        run(f"umount {root_subvol}")
+        pathlib.Path(root_subvol).rmdir()
+
+
+def btrfs_delete_snapshot(volume_id, name):
+    import btrfsutil
+
+    with mount_root_subvol(volume_id) as root_subvol:
+        snapshots_dir = f"{root_subvol}/.snapshots"
+        snapshot_path = f"{snapshots_dir}/{name}"
+        btrfsutil.delete_subvolume(snapshot_path)
+
+
+def btrfs_create_snapshot(volume_id, name):
+    import btrfsutil
+    import time
+    import pathlib
+
+    # TODO: check fstype
+
+    with mount_root_subvol(volume_id) as root_subvol:
+        default_subvol_id = btrfsutil.get_default_subvolume(root_subvol)
+        default_subvol = btrfsutil.subvolume_path(root_subvol, default_subvol_id)
+        default_subvol = f"{root_subvol}/{default_subvol}"
+
+        snapshots_dir = f"{root_subvol}/.snapshots"
+        pathlib.Path(snapshots_dir).mkdir(parents=True, exist_ok=True)
+
+        snapshot_subvol = f"{snapshots_dir}/{name}"
+        btrfsutil.create_snapshot(default_subvol, snapshot_subvol, read_only=True)
+
+    snapshot_id = f"{volume_id}/{name}"
+    creation_time_ns = time.time_ns()
+    return snapshot_id, creation_time_ns

requirements.in

@@ -5,3 +5,4 @@ pyyaml
 pykube-ng
 munch
 prometheus_client
+#https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python