Support creating snapshots from btrfs volumes
This commit is contained in:
parent
c978b3290b
commit
0a130f42ff
@ -1,11 +1,18 @@
|
|||||||
|
FROM python:3.10-slim-buster as builder
|
||||||
|
RUN apt-get update && apt-get install -y build-essential libbtrfsutil-dev
|
||||||
|
RUN pip wheel -w /wheels "https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python"
|
||||||
|
|
||||||
FROM python:3.10-slim-buster
|
FROM python:3.10-slim-buster
|
||||||
|
|
||||||
WORKDIR /app/
|
WORKDIR /app/
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y e2fsprogs btrfs-progs xfsprogs && \
|
apt-get install -y e2fsprogs btrfs-progs libbtrfsutil1 xfsprogs && \
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=builder /wheels/ /wheels/
|
||||||
|
RUN pip install /wheels/*
|
||||||
|
|
||||||
ENV PIP_NO_CACHE_DIR 1
|
ENV PIP_NO_CACHE_DIR 1
|
||||||
ADD ./requirements.txt ./
|
ADD ./requirements.txt ./
|
||||||
RUN pip install -r ./requirements.txt
|
RUN pip install -r ./requirements.txt
|
||||||
|
@ -48,7 +48,7 @@ Features
|
|||||||
- [ ] Online shrinking: If fs supports it (e.g. btrfs)
|
- [ ] Online shrinking: If fs supports it (e.g. btrfs)
|
||||||
- [ ] Offline expansion/shrinking
|
- [ ] Offline expansion/shrinking
|
||||||
- [ ] Ephemeral inline volume
|
- [ ] Ephemeral inline volume
|
||||||
- [ ] Snapshots: If the fs supports it (e.g. btrfs)
|
- [x] Filesystem-level snapshots: `btrfs` supported
|
||||||
|
|
||||||
Motivation
|
Motivation
|
||||||
---
|
---
|
||||||
|
33
bd2fs.py
33
bd2fs.py
@ -10,6 +10,8 @@ from csi.csi_pb2 import (
|
|||||||
NodeExpandVolumeRequest,
|
NodeExpandVolumeRequest,
|
||||||
CreateVolumeRequest,
|
CreateVolumeRequest,
|
||||||
)
|
)
|
||||||
|
from google.protobuf.timestamp_pb2 import Timestamp
|
||||||
|
|
||||||
from declarative import (
|
from declarative import (
|
||||||
be_mounted,
|
be_mounted,
|
||||||
be_unmounted,
|
be_unmounted,
|
||||||
@ -18,6 +20,8 @@ from declarative import (
|
|||||||
be_fs_expanded,
|
be_fs_expanded,
|
||||||
)
|
)
|
||||||
from fs_util import path_stats, mountpoint_to_dev
|
from fs_util import path_stats, mountpoint_to_dev
|
||||||
|
from orchestrator.k8s import volume_to_node, run_on_node
|
||||||
|
from remote import btrfs_create_snapshot, btrfs_delete_snapshot
|
||||||
from util import log_grpc_request
|
from util import log_grpc_request
|
||||||
|
|
||||||
|
|
||||||
@ -218,3 +222,32 @@ class Bd2FsControllerServicer(csi_pb2_grpc.ControllerServicer):
|
|||||||
response = self.bds.ControllerExpandVolume(request, context)
|
response = self.bds.ControllerExpandVolume(request, context)
|
||||||
assert response.node_expansion_required
|
assert response.node_expansion_required
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
@log_grpc_request
|
||||||
|
def CreateSnapshot(self, request: csi_pb2.CreateSnapshotRequest, context):
|
||||||
|
volume_id = request.source_volume_id
|
||||||
|
name = request.name
|
||||||
|
|
||||||
|
snapshot_id, creation_time_ns = btrfs_create_snapshot(
|
||||||
|
volume_id=volume_id, name=name
|
||||||
|
)
|
||||||
|
|
||||||
|
nano = 10**9
|
||||||
|
return csi_pb2.CreateSnapshotResponse(
|
||||||
|
snapshot=csi_pb2.Snapshot(
|
||||||
|
size_bytes=0,
|
||||||
|
snapshot_id=snapshot_id,
|
||||||
|
source_volume_id=volume_id,
|
||||||
|
creation_time=Timestamp(
|
||||||
|
seconds=creation_time_ns // nano, nanos=creation_time_ns % nano
|
||||||
|
),
|
||||||
|
ready_to_use=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
@log_grpc_request
|
||||||
|
def DeleteSnapshot(self, request: csi_pb2.DeleteSnapshotRequest, context):
|
||||||
|
snapshot_id = request.snapshot_id
|
||||||
|
volume_id, name = snapshot_id.rsplit("/", 1)
|
||||||
|
btrfs_delete_snapshot(volume_id=volume_id, name=name)
|
||||||
|
return csi_pb2.DeleteSnapshotResponse()
|
||||||
|
@ -124,3 +124,34 @@ roleRef:
|
|||||||
kind: ClusterRole
|
kind: ClusterRole
|
||||||
name: {{ include "rawfile-csi.fullname" . }}-resizer
|
name: {{ include "rawfile-csi.fullname" . }}-resizer
|
||||||
apiGroup: rbac.authorization.k8s.io
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
---
|
||||||
|
kind: ClusterRole
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["events"]
|
||||||
|
verbs: ["list", "watch", "create", "update", "patch"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshotclasses"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshotcontents"]
|
||||||
|
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshotcontents/status"]
|
||||||
|
verbs: ["update", "patch"]
|
||||||
|
---
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: {{ include "rawfile-csi.fullname" . }}-driver
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
@ -153,3 +153,20 @@ spec:
|
|||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: socket-dir
|
- name: socket-dir
|
||||||
mountPath: /csi
|
mountPath: /csi
|
||||||
|
- name: external-snapshotter
|
||||||
|
image: gcr.io/k8s-staging-sig-storage/csi-snapshotter:v5.0.1
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- "--csi-address=$(ADDRESS)"
|
||||||
|
- "--node-deployment=true"
|
||||||
|
- "--extra-create-metadata=true"
|
||||||
|
env:
|
||||||
|
- name: ADDRESS
|
||||||
|
value: /csi/csi.sock
|
||||||
|
- name: NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
volumeMounts:
|
||||||
|
- name: socket-dir
|
||||||
|
mountPath: /csi
|
||||||
|
@ -141,6 +141,7 @@ class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer):
|
|||||||
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)),
|
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)),
|
||||||
Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)),
|
Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)),
|
||||||
Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
|
Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
|
||||||
|
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_SNAPSHOT)),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
55
remote.py
55
remote.py
@ -1,3 +1,5 @@
|
|||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
from util import remote_fn
|
from util import remote_fn
|
||||||
|
|
||||||
|
|
||||||
@ -81,3 +83,56 @@ def expand_rawfile(volume_id, size):
|
|||||||
{"size": size},
|
{"size": size},
|
||||||
)
|
)
|
||||||
run(f"truncate -s {size} {img_file}")
|
run(f"truncate -s {size} {img_file}")
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def mount_root_subvol(volume_id):
|
||||||
|
import tempfile
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
import rawfile_util
|
||||||
|
from util import run
|
||||||
|
|
||||||
|
root_subvol = tempfile.mkdtemp(prefix="rawfile-")
|
||||||
|
|
||||||
|
img_file = rawfile_util.img_file(volume_id)
|
||||||
|
loop_dev = rawfile_util.attach_loop(img_file)
|
||||||
|
|
||||||
|
run(f"mount -t btrfs -o subvolid=0 {loop_dev} {root_subvol}")
|
||||||
|
try:
|
||||||
|
yield root_subvol
|
||||||
|
finally:
|
||||||
|
run(f"umount {root_subvol}")
|
||||||
|
pathlib.Path(root_subvol).rmdir()
|
||||||
|
|
||||||
|
|
||||||
|
def btrfs_delete_snapshot(volume_id, name):
|
||||||
|
import btrfsutil
|
||||||
|
|
||||||
|
with mount_root_subvol(volume_id) as root_subvol:
|
||||||
|
snapshots_dir = f"{root_subvol}/.snapshots"
|
||||||
|
snapshot_path = f"{snapshots_dir}/{name}"
|
||||||
|
btrfsutil.delete_subvolume(snapshot_path)
|
||||||
|
|
||||||
|
|
||||||
|
def btrfs_create_snapshot(volume_id, name):
|
||||||
|
import btrfsutil
|
||||||
|
import time
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
# TODO: check fstype
|
||||||
|
|
||||||
|
with mount_root_subvol(volume_id) as root_subvol:
|
||||||
|
default_subvol_id = btrfsutil.get_default_subvolume(root_subvol)
|
||||||
|
default_subvol = btrfsutil.subvolume_path(root_subvol, default_subvol_id)
|
||||||
|
default_subvol = f"{root_subvol}/{default_subvol}"
|
||||||
|
|
||||||
|
snapshots_dir = f"{root_subvol}/.snapshots"
|
||||||
|
pathlib.Path(snapshots_dir).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
snapshot_subvol = f"{snapshots_dir}/{name}"
|
||||||
|
btrfsutil.create_snapshot(default_subvol, snapshot_subvol, read_only=True)
|
||||||
|
|
||||||
|
snapshot_id = f"{volume_id}/{name}"
|
||||||
|
creation_time_ns = time.time_ns()
|
||||||
|
return snapshot_id, creation_time_ns
|
||||||
|
@ -5,3 +5,4 @@ pyyaml
|
|||||||
pykube-ng
|
pykube-ng
|
||||||
munch
|
munch
|
||||||
prometheus_client
|
prometheus_client
|
||||||
|
#https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python
|
||||||
|
Loading…
Reference in New Issue
Block a user