From 0a130f42ffd8920949edc7652e1a55c6deba578d Mon Sep 17 00:00:00 2001 From: Mehran Kholdi Date: Thu, 17 Feb 2022 17:26:31 +0330 Subject: [PATCH] Support creating snapshots from btrfs volumes --- Dockerfile | 9 ++- README.md | 2 +- bd2fs.py | 33 +++++++++++ .../charts/rawfile-csi/templates/00-rbac.yaml | 31 +++++++++++ .../rawfile-csi/templates/01-node-plugin.yaml | 17 ++++++ rawfile_servicer.py | 1 + remote.py | 55 +++++++++++++++++++ requirements.in | 1 + 8 files changed, 147 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 565db65..feddb06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,18 @@ +FROM python:3.10-slim-buster as builder +RUN apt-get update && apt-get install -y build-essential libbtrfsutil-dev +RUN pip wheel -w /wheels "https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python" + FROM python:3.10-slim-buster WORKDIR /app/ RUN apt-get update && \ - apt-get install -y e2fsprogs btrfs-progs xfsprogs && \ + apt-get install -y e2fsprogs btrfs-progs libbtrfsutil1 xfsprogs && \ rm -rf /var/lib/apt/lists/* +COPY --from=builder /wheels/ /wheels/ +RUN pip install /wheels/* + ENV PIP_NO_CACHE_DIR 1 ADD ./requirements.txt ./ RUN pip install -r ./requirements.txt diff --git a/README.md b/README.md index 36a956c..d49c847 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Features - [ ] Online shrinking: If fs supports it (e.g. btrfs) - [ ] Offline expansion/shrinking - [ ] Ephemeral inline volume -- [ ] Snapshots: If the fs supports it (e.g. btrfs) +- [x] Filesystem-level snapshots: `btrfs` supported Motivation --- diff --git a/bd2fs.py b/bd2fs.py index bc9223a..5a88edf 100644 --- a/bd2fs.py +++ b/bd2fs.py @@ -10,6 +10,8 @@ from csi.csi_pb2 import ( NodeExpandVolumeRequest, CreateVolumeRequest, ) +from google.protobuf.timestamp_pb2 import Timestamp + from declarative import ( be_mounted, be_unmounted, @@ -18,6 +20,8 @@ from declarative import ( be_fs_expanded, ) from fs_util import path_stats, mountpoint_to_dev +from orchestrator.k8s import volume_to_node, run_on_node +from remote import btrfs_create_snapshot, btrfs_delete_snapshot from util import log_grpc_request @@ -218,3 +222,32 @@ class Bd2FsControllerServicer(csi_pb2_grpc.ControllerServicer): response = self.bds.ControllerExpandVolume(request, context) assert response.node_expansion_required return response + + @log_grpc_request + def CreateSnapshot(self, request: csi_pb2.CreateSnapshotRequest, context): + volume_id = request.source_volume_id + name = request.name + + snapshot_id, creation_time_ns = btrfs_create_snapshot( + volume_id=volume_id, name=name + ) + + nano = 10**9 + return csi_pb2.CreateSnapshotResponse( + snapshot=csi_pb2.Snapshot( + size_bytes=0, + snapshot_id=snapshot_id, + source_volume_id=volume_id, + creation_time=Timestamp( + seconds=creation_time_ns // nano, nanos=creation_time_ns % nano + ), + ready_to_use=True, + ) + ) + + @log_grpc_request + def DeleteSnapshot(self, request: csi_pb2.DeleteSnapshotRequest, context): + snapshot_id = request.snapshot_id + volume_id, name = snapshot_id.rsplit("/", 1) + btrfs_delete_snapshot(volume_id=volume_id, name=name) + return csi_pb2.DeleteSnapshotResponse() diff --git a/deploy/charts/rawfile-csi/templates/00-rbac.yaml b/deploy/charts/rawfile-csi/templates/00-rbac.yaml index 3830937..4742a15 100644 --- a/deploy/charts/rawfile-csi/templates/00-rbac.yaml +++ b/deploy/charts/rawfile-csi/templates/00-rbac.yaml @@ -124,3 +124,34 @@ roleRef: kind: ClusterRole name: {{ include "rawfile-csi.fullname" . }}-resizer apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "rawfile-csi.fullname" . }}-snapshotter +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "rawfile-csi.fullname" . }}-snapshotter +subjects: + - kind: ServiceAccount + name: {{ include "rawfile-csi.fullname" . }}-driver + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "rawfile-csi.fullname" . }}-snapshotter + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml b/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml index ae20e48..06e1d91 100644 --- a/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml +++ b/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml @@ -153,3 +153,20 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi + - name: external-snapshotter + image: gcr.io/k8s-staging-sig-storage/csi-snapshotter:v5.0.1 + imagePullPolicy: IfNotPresent + args: + - "--csi-address=$(ADDRESS)" + - "--node-deployment=true" + - "--extra-create-metadata=true" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi diff --git a/rawfile_servicer.py b/rawfile_servicer.py index 869fcbf..3823cdc 100644 --- a/rawfile_servicer.py +++ b/rawfile_servicer.py @@ -141,6 +141,7 @@ class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer): Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)), Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)), Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)), + Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_SNAPSHOT)), ] ) diff --git a/remote.py b/remote.py index b77d75c..f24f3fe 100644 --- a/remote.py +++ b/remote.py @@ -1,3 +1,5 @@ +from contextlib import contextmanager + from util import remote_fn @@ -81,3 +83,56 @@ def expand_rawfile(volume_id, size): {"size": size}, ) run(f"truncate -s {size} {img_file}") + + +@contextmanager +def mount_root_subvol(volume_id): + import tempfile + import pathlib + + import rawfile_util + from util import run + + root_subvol = tempfile.mkdtemp(prefix="rawfile-") + + img_file = rawfile_util.img_file(volume_id) + loop_dev = rawfile_util.attach_loop(img_file) + + run(f"mount -t btrfs -o subvolid=0 {loop_dev} {root_subvol}") + try: + yield root_subvol + finally: + run(f"umount {root_subvol}") + pathlib.Path(root_subvol).rmdir() + + +def btrfs_delete_snapshot(volume_id, name): + import btrfsutil + + with mount_root_subvol(volume_id) as root_subvol: + snapshots_dir = f"{root_subvol}/.snapshots" + snapshot_path = f"{snapshots_dir}/{name}" + btrfsutil.delete_subvolume(snapshot_path) + + +def btrfs_create_snapshot(volume_id, name): + import btrfsutil + import time + import pathlib + + # TODO: check fstype + + with mount_root_subvol(volume_id) as root_subvol: + default_subvol_id = btrfsutil.get_default_subvolume(root_subvol) + default_subvol = btrfsutil.subvolume_path(root_subvol, default_subvol_id) + default_subvol = f"{root_subvol}/{default_subvol}" + + snapshots_dir = f"{root_subvol}/.snapshots" + pathlib.Path(snapshots_dir).mkdir(parents=True, exist_ok=True) + + snapshot_subvol = f"{snapshots_dir}/{name}" + btrfsutil.create_snapshot(default_subvol, snapshot_subvol, read_only=True) + + snapshot_id = f"{volume_id}/{name}" + creation_time_ns = time.time_ns() + return snapshot_id, creation_time_ns diff --git a/requirements.in b/requirements.in index e56c24c..5cfe359 100644 --- a/requirements.in +++ b/requirements.in @@ -5,3 +5,4 @@ pyyaml pykube-ng munch prometheus_client +#https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python