Support creating snapshots from btrfs volumes
This commit is contained in:
parent
c978b3290b
commit
0a130f42ff
@ -1,11 +1,18 @@
|
||||
FROM python:3.10-slim-buster as builder
|
||||
RUN apt-get update && apt-get install -y build-essential libbtrfsutil-dev
|
||||
RUN pip wheel -w /wheels "https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python"
|
||||
|
||||
FROM python:3.10-slim-buster
|
||||
|
||||
WORKDIR /app/
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y e2fsprogs btrfs-progs xfsprogs && \
|
||||
apt-get install -y e2fsprogs btrfs-progs libbtrfsutil1 xfsprogs && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /wheels/ /wheels/
|
||||
RUN pip install /wheels/*
|
||||
|
||||
ENV PIP_NO_CACHE_DIR 1
|
||||
ADD ./requirements.txt ./
|
||||
RUN pip install -r ./requirements.txt
|
||||
|
@ -48,7 +48,7 @@ Features
|
||||
- [ ] Online shrinking: If fs supports it (e.g. btrfs)
|
||||
- [ ] Offline expansion/shrinking
|
||||
- [ ] Ephemeral inline volume
|
||||
- [ ] Snapshots: If the fs supports it (e.g. btrfs)
|
||||
- [x] Filesystem-level snapshots: `btrfs` supported
|
||||
|
||||
Motivation
|
||||
---
|
||||
|
33
bd2fs.py
33
bd2fs.py
@ -10,6 +10,8 @@ from csi.csi_pb2 import (
|
||||
NodeExpandVolumeRequest,
|
||||
CreateVolumeRequest,
|
||||
)
|
||||
from google.protobuf.timestamp_pb2 import Timestamp
|
||||
|
||||
from declarative import (
|
||||
be_mounted,
|
||||
be_unmounted,
|
||||
@ -18,6 +20,8 @@ from declarative import (
|
||||
be_fs_expanded,
|
||||
)
|
||||
from fs_util import path_stats, mountpoint_to_dev
|
||||
from orchestrator.k8s import volume_to_node, run_on_node
|
||||
from remote import btrfs_create_snapshot, btrfs_delete_snapshot
|
||||
from util import log_grpc_request
|
||||
|
||||
|
||||
@ -218,3 +222,32 @@ class Bd2FsControllerServicer(csi_pb2_grpc.ControllerServicer):
|
||||
response = self.bds.ControllerExpandVolume(request, context)
|
||||
assert response.node_expansion_required
|
||||
return response
|
||||
|
||||
@log_grpc_request
|
||||
def CreateSnapshot(self, request: csi_pb2.CreateSnapshotRequest, context):
|
||||
volume_id = request.source_volume_id
|
||||
name = request.name
|
||||
|
||||
snapshot_id, creation_time_ns = btrfs_create_snapshot(
|
||||
volume_id=volume_id, name=name
|
||||
)
|
||||
|
||||
nano = 10**9
|
||||
return csi_pb2.CreateSnapshotResponse(
|
||||
snapshot=csi_pb2.Snapshot(
|
||||
size_bytes=0,
|
||||
snapshot_id=snapshot_id,
|
||||
source_volume_id=volume_id,
|
||||
creation_time=Timestamp(
|
||||
seconds=creation_time_ns // nano, nanos=creation_time_ns % nano
|
||||
),
|
||||
ready_to_use=True,
|
||||
)
|
||||
)
|
||||
|
||||
@log_grpc_request
|
||||
def DeleteSnapshot(self, request: csi_pb2.DeleteSnapshotRequest, context):
|
||||
snapshot_id = request.snapshot_id
|
||||
volume_id, name = snapshot_id.rsplit("/", 1)
|
||||
btrfs_delete_snapshot(volume_id=volume_id, name=name)
|
||||
return csi_pb2.DeleteSnapshotResponse()
|
||||
|
@ -124,3 +124,34 @@ roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "rawfile-csi.fullname" . }}-resizer
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents/status"]
|
||||
verbs: ["update", "patch"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "rawfile-csi.fullname" . }}-driver
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
@ -153,3 +153,20 @@ spec:
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: external-snapshotter
|
||||
image: gcr.io/k8s-staging-sig-storage/csi-snapshotter:v5.0.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--node-deployment=true"
|
||||
- "--extra-create-metadata=true"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
|
@ -141,6 +141,7 @@ class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer):
|
||||
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)),
|
||||
Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)),
|
||||
Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
|
||||
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_SNAPSHOT)),
|
||||
]
|
||||
)
|
||||
|
||||
|
55
remote.py
55
remote.py
@ -1,3 +1,5 @@
|
||||
from contextlib import contextmanager
|
||||
|
||||
from util import remote_fn
|
||||
|
||||
|
||||
@ -81,3 +83,56 @@ def expand_rawfile(volume_id, size):
|
||||
{"size": size},
|
||||
)
|
||||
run(f"truncate -s {size} {img_file}")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def mount_root_subvol(volume_id):
|
||||
import tempfile
|
||||
import pathlib
|
||||
|
||||
import rawfile_util
|
||||
from util import run
|
||||
|
||||
root_subvol = tempfile.mkdtemp(prefix="rawfile-")
|
||||
|
||||
img_file = rawfile_util.img_file(volume_id)
|
||||
loop_dev = rawfile_util.attach_loop(img_file)
|
||||
|
||||
run(f"mount -t btrfs -o subvolid=0 {loop_dev} {root_subvol}")
|
||||
try:
|
||||
yield root_subvol
|
||||
finally:
|
||||
run(f"umount {root_subvol}")
|
||||
pathlib.Path(root_subvol).rmdir()
|
||||
|
||||
|
||||
def btrfs_delete_snapshot(volume_id, name):
|
||||
import btrfsutil
|
||||
|
||||
with mount_root_subvol(volume_id) as root_subvol:
|
||||
snapshots_dir = f"{root_subvol}/.snapshots"
|
||||
snapshot_path = f"{snapshots_dir}/{name}"
|
||||
btrfsutil.delete_subvolume(snapshot_path)
|
||||
|
||||
|
||||
def btrfs_create_snapshot(volume_id, name):
|
||||
import btrfsutil
|
||||
import time
|
||||
import pathlib
|
||||
|
||||
# TODO: check fstype
|
||||
|
||||
with mount_root_subvol(volume_id) as root_subvol:
|
||||
default_subvol_id = btrfsutil.get_default_subvolume(root_subvol)
|
||||
default_subvol = btrfsutil.subvolume_path(root_subvol, default_subvol_id)
|
||||
default_subvol = f"{root_subvol}/{default_subvol}"
|
||||
|
||||
snapshots_dir = f"{root_subvol}/.snapshots"
|
||||
pathlib.Path(snapshots_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
snapshot_subvol = f"{snapshots_dir}/{name}"
|
||||
btrfsutil.create_snapshot(default_subvol, snapshot_subvol, read_only=True)
|
||||
|
||||
snapshot_id = f"{volume_id}/{name}"
|
||||
creation_time_ns = time.time_ns()
|
||||
return snapshot_id, creation_time_ns
|
||||
|
@ -5,3 +5,4 @@ pyyaml
|
||||
pykube-ng
|
||||
munch
|
||||
prometheus_client
|
||||
#https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python
|
||||
|
Loading…
Reference in New Issue
Block a user