Support creating snapshots from btrfs volumes

This commit is contained in:
Mehran Kholdi 2022-02-17 17:26:31 +03:30
parent c978b3290b
commit 0a130f42ff
8 changed files with 147 additions and 2 deletions

View File

@ -1,11 +1,18 @@
FROM python:3.10-slim-buster as builder
RUN apt-get update && apt-get install -y build-essential libbtrfsutil-dev
RUN pip wheel -w /wheels "https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python"
FROM python:3.10-slim-buster FROM python:3.10-slim-buster
WORKDIR /app/ WORKDIR /app/
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y e2fsprogs btrfs-progs xfsprogs && \ apt-get install -y e2fsprogs btrfs-progs libbtrfsutil1 xfsprogs && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
COPY --from=builder /wheels/ /wheels/
RUN pip install /wheels/*
ENV PIP_NO_CACHE_DIR 1 ENV PIP_NO_CACHE_DIR 1
ADD ./requirements.txt ./ ADD ./requirements.txt ./
RUN pip install -r ./requirements.txt RUN pip install -r ./requirements.txt

View File

@ -48,7 +48,7 @@ Features
- [ ] Online shrinking: If fs supports it (e.g. btrfs) - [ ] Online shrinking: If fs supports it (e.g. btrfs)
- [ ] Offline expansion/shrinking - [ ] Offline expansion/shrinking
- [ ] Ephemeral inline volume - [ ] Ephemeral inline volume
- [ ] Snapshots: If the fs supports it (e.g. btrfs) - [x] Filesystem-level snapshots: `btrfs` supported
Motivation Motivation
--- ---

View File

@ -10,6 +10,8 @@ from csi.csi_pb2 import (
NodeExpandVolumeRequest, NodeExpandVolumeRequest,
CreateVolumeRequest, CreateVolumeRequest,
) )
from google.protobuf.timestamp_pb2 import Timestamp
from declarative import ( from declarative import (
be_mounted, be_mounted,
be_unmounted, be_unmounted,
@ -18,6 +20,8 @@ from declarative import (
be_fs_expanded, be_fs_expanded,
) )
from fs_util import path_stats, mountpoint_to_dev from fs_util import path_stats, mountpoint_to_dev
from orchestrator.k8s import volume_to_node, run_on_node
from remote import btrfs_create_snapshot, btrfs_delete_snapshot
from util import log_grpc_request from util import log_grpc_request
@ -218,3 +222,32 @@ class Bd2FsControllerServicer(csi_pb2_grpc.ControllerServicer):
response = self.bds.ControllerExpandVolume(request, context) response = self.bds.ControllerExpandVolume(request, context)
assert response.node_expansion_required assert response.node_expansion_required
return response return response
@log_grpc_request
def CreateSnapshot(self, request: csi_pb2.CreateSnapshotRequest, context):
volume_id = request.source_volume_id
name = request.name
snapshot_id, creation_time_ns = btrfs_create_snapshot(
volume_id=volume_id, name=name
)
nano = 10**9
return csi_pb2.CreateSnapshotResponse(
snapshot=csi_pb2.Snapshot(
size_bytes=0,
snapshot_id=snapshot_id,
source_volume_id=volume_id,
creation_time=Timestamp(
seconds=creation_time_ns // nano, nanos=creation_time_ns % nano
),
ready_to_use=True,
)
)
@log_grpc_request
def DeleteSnapshot(self, request: csi_pb2.DeleteSnapshotRequest, context):
snapshot_id = request.snapshot_id
volume_id, name = snapshot_id.rsplit("/", 1)
btrfs_delete_snapshot(volume_id=volume_id, name=name)
return csi_pb2.DeleteSnapshotResponse()

View File

@ -124,3 +124,34 @@ roleRef:
kind: ClusterRole kind: ClusterRole
name: {{ include "rawfile-csi.fullname" . }}-resizer name: {{ include "rawfile-csi.fullname" . }}-resizer
apiGroup: rbac.authorization.k8s.io apiGroup: rbac.authorization.k8s.io
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents/status"]
verbs: ["update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
subjects:
- kind: ServiceAccount
name: {{ include "rawfile-csi.fullname" . }}-driver
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
apiGroup: rbac.authorization.k8s.io

View File

@ -153,3 +153,20 @@ spec:
volumeMounts: volumeMounts:
- name: socket-dir - name: socket-dir
mountPath: /csi mountPath: /csi
- name: external-snapshotter
image: gcr.io/k8s-staging-sig-storage/csi-snapshotter:v5.0.1
imagePullPolicy: IfNotPresent
args:
- "--csi-address=$(ADDRESS)"
- "--node-deployment=true"
- "--extra-create-metadata=true"
env:
- name: ADDRESS
value: /csi/csi.sock
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: socket-dir
mountPath: /csi

View File

@ -141,6 +141,7 @@ class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer):
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)), Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)),
Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)), Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)),
Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)), Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_SNAPSHOT)),
] ]
) )

View File

@ -1,3 +1,5 @@
from contextlib import contextmanager
from util import remote_fn from util import remote_fn
@ -81,3 +83,56 @@ def expand_rawfile(volume_id, size):
{"size": size}, {"size": size},
) )
run(f"truncate -s {size} {img_file}") run(f"truncate -s {size} {img_file}")
@contextmanager
def mount_root_subvol(volume_id):
import tempfile
import pathlib
import rawfile_util
from util import run
root_subvol = tempfile.mkdtemp(prefix="rawfile-")
img_file = rawfile_util.img_file(volume_id)
loop_dev = rawfile_util.attach_loop(img_file)
run(f"mount -t btrfs -o subvolid=0 {loop_dev} {root_subvol}")
try:
yield root_subvol
finally:
run(f"umount {root_subvol}")
pathlib.Path(root_subvol).rmdir()
def btrfs_delete_snapshot(volume_id, name):
import btrfsutil
with mount_root_subvol(volume_id) as root_subvol:
snapshots_dir = f"{root_subvol}/.snapshots"
snapshot_path = f"{snapshots_dir}/{name}"
btrfsutil.delete_subvolume(snapshot_path)
def btrfs_create_snapshot(volume_id, name):
import btrfsutil
import time
import pathlib
# TODO: check fstype
with mount_root_subvol(volume_id) as root_subvol:
default_subvol_id = btrfsutil.get_default_subvolume(root_subvol)
default_subvol = btrfsutil.subvolume_path(root_subvol, default_subvol_id)
default_subvol = f"{root_subvol}/{default_subvol}"
snapshots_dir = f"{root_subvol}/.snapshots"
pathlib.Path(snapshots_dir).mkdir(parents=True, exist_ok=True)
snapshot_subvol = f"{snapshots_dir}/{name}"
btrfsutil.create_snapshot(default_subvol, snapshot_subvol, read_only=True)
snapshot_id = f"{volume_id}/{name}"
creation_time_ns = time.time_ns()
return snapshot_id, creation_time_ns

View File

@ -5,3 +5,4 @@ pyyaml
pykube-ng pykube-ng
munch munch
prometheus_client prometheus_client
#https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python