Compare commits
7 Commits
Author | SHA1 | Date |
---|---|---|
Lauri Võsandi | 6648d0070c | |
Lauri Võsandi | 439954e3ed | |
Mehran Kholdi | 4e0a4fe698 | |
Mehran Kholdi | 0a130f42ff | |
Mehran Kholdi | c978b3290b | |
Mehran Kholdi | 22f2fb1628 | |
Mehran Kholdi | 2d1fa49b2a |
|
@ -0,0 +1,2 @@
|
||||||
|
kind: template
|
||||||
|
load: docker-multiarch.yaml
|
11
Dockerfile
11
Dockerfile
|
@ -1,11 +1,18 @@
|
||||||
FROM python:3.9-slim-buster
|
FROM python:3.10-slim-buster as builder
|
||||||
|
RUN apt-get update && apt-get install -y build-essential libbtrfsutil-dev
|
||||||
|
RUN pip wheel -w /wheels "https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python"
|
||||||
|
|
||||||
|
FROM python:3.10-slim-buster
|
||||||
|
|
||||||
WORKDIR /app/
|
WORKDIR /app/
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y e2fsprogs btrfs-progs xfsprogs && \
|
apt-get install -y e2fsprogs btrfs-progs libbtrfsutil1 xfsprogs && \
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=builder /wheels/ /wheels/
|
||||||
|
RUN pip install /wheels/*
|
||||||
|
|
||||||
ENV PIP_NO_CACHE_DIR 1
|
ENV PIP_NO_CACHE_DIR 1
|
||||||
ADD ./requirements.txt ./
|
ADD ./requirements.txt ./
|
||||||
RUN pip install -r ./requirements.txt
|
RUN pip install -r ./requirements.txt
|
||||||
|
|
|
@ -48,7 +48,7 @@ Features
|
||||||
- [ ] Online shrinking: If fs supports it (e.g. btrfs)
|
- [ ] Online shrinking: If fs supports it (e.g. btrfs)
|
||||||
- [ ] Offline expansion/shrinking
|
- [ ] Offline expansion/shrinking
|
||||||
- [ ] Ephemeral inline volume
|
- [ ] Ephemeral inline volume
|
||||||
- [ ] Snapshots: If the fs supports it (e.g. btrfs)
|
- [x] Filesystem-level snapshots: `btrfs` supported
|
||||||
|
|
||||||
Motivation
|
Motivation
|
||||||
---
|
---
|
||||||
|
|
40
bd2fs.py
40
bd2fs.py
|
@ -1,7 +1,6 @@
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import grpc
|
import grpc
|
||||||
|
|
||||||
from csi import csi_pb2, csi_pb2_grpc
|
from csi import csi_pb2, csi_pb2_grpc
|
||||||
from csi.csi_pb2 import (
|
from csi.csi_pb2 import (
|
||||||
NodeStageVolumeRequest,
|
NodeStageVolumeRequest,
|
||||||
|
@ -11,6 +10,8 @@ from csi.csi_pb2 import (
|
||||||
NodeExpandVolumeRequest,
|
NodeExpandVolumeRequest,
|
||||||
CreateVolumeRequest,
|
CreateVolumeRequest,
|
||||||
)
|
)
|
||||||
|
from google.protobuf.timestamp_pb2 import Timestamp
|
||||||
|
|
||||||
from declarative import (
|
from declarative import (
|
||||||
be_mounted,
|
be_mounted,
|
||||||
be_unmounted,
|
be_unmounted,
|
||||||
|
@ -19,6 +20,8 @@ from declarative import (
|
||||||
be_fs_expanded,
|
be_fs_expanded,
|
||||||
)
|
)
|
||||||
from fs_util import path_stats, mountpoint_to_dev
|
from fs_util import path_stats, mountpoint_to_dev
|
||||||
|
from orchestrator.k8s import volume_to_node, run_on_node
|
||||||
|
from remote import btrfs_create_snapshot, btrfs_delete_snapshot
|
||||||
from util import log_grpc_request
|
from util import log_grpc_request
|
||||||
|
|
||||||
|
|
||||||
|
@ -30,7 +33,7 @@ def get_fs(request):
|
||||||
|
|
||||||
|
|
||||||
class Bd2FsIdentityServicer(csi_pb2_grpc.IdentityServicer):
|
class Bd2FsIdentityServicer(csi_pb2_grpc.IdentityServicer):
|
||||||
def __init__(self, bds):
|
def __init__(self, bds: csi_pb2_grpc.IdentityServicer):
|
||||||
self.bds = bds
|
self.bds = bds
|
||||||
|
|
||||||
@log_grpc_request
|
@log_grpc_request
|
||||||
|
@ -47,7 +50,7 @@ class Bd2FsIdentityServicer(csi_pb2_grpc.IdentityServicer):
|
||||||
|
|
||||||
|
|
||||||
class Bd2FsNodeServicer(csi_pb2_grpc.NodeServicer):
|
class Bd2FsNodeServicer(csi_pb2_grpc.NodeServicer):
|
||||||
def __init__(self, bds):
|
def __init__(self, bds: csi_pb2_grpc.NodeServicer):
|
||||||
self.bds = bds
|
self.bds = bds
|
||||||
|
|
||||||
# @log_grpc_request
|
# @log_grpc_request
|
||||||
|
@ -167,7 +170,7 @@ class Bd2FsNodeServicer(csi_pb2_grpc.NodeServicer):
|
||||||
|
|
||||||
|
|
||||||
class Bd2FsControllerServicer(csi_pb2_grpc.ControllerServicer):
|
class Bd2FsControllerServicer(csi_pb2_grpc.ControllerServicer):
|
||||||
def __init__(self, bds):
|
def __init__(self, bds: csi_pb2_grpc.ControllerServicer):
|
||||||
self.bds = bds
|
self.bds = bds
|
||||||
|
|
||||||
@log_grpc_request
|
@log_grpc_request
|
||||||
|
@ -219,3 +222,32 @@ class Bd2FsControllerServicer(csi_pb2_grpc.ControllerServicer):
|
||||||
response = self.bds.ControllerExpandVolume(request, context)
|
response = self.bds.ControllerExpandVolume(request, context)
|
||||||
assert response.node_expansion_required
|
assert response.node_expansion_required
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
@log_grpc_request
|
||||||
|
def CreateSnapshot(self, request: csi_pb2.CreateSnapshotRequest, context):
|
||||||
|
volume_id = request.source_volume_id
|
||||||
|
name = request.name
|
||||||
|
|
||||||
|
snapshot_id, creation_time_ns = btrfs_create_snapshot(
|
||||||
|
volume_id=volume_id, name=name
|
||||||
|
)
|
||||||
|
|
||||||
|
nano = 10**9
|
||||||
|
return csi_pb2.CreateSnapshotResponse(
|
||||||
|
snapshot=csi_pb2.Snapshot(
|
||||||
|
size_bytes=0,
|
||||||
|
snapshot_id=snapshot_id,
|
||||||
|
source_volume_id=volume_id,
|
||||||
|
creation_time=Timestamp(
|
||||||
|
seconds=creation_time_ns // nano, nanos=creation_time_ns % nano
|
||||||
|
),
|
||||||
|
ready_to_use=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
@log_grpc_request
|
||||||
|
def DeleteSnapshot(self, request: csi_pb2.DeleteSnapshotRequest, context):
|
||||||
|
snapshot_id = request.snapshot_id
|
||||||
|
volume_id, name = snapshot_id.rsplit("/", 1)
|
||||||
|
btrfs_delete_snapshot(volume_id=volume_id, name=name)
|
||||||
|
return csi_pb2.DeleteSnapshotResponse()
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import os
|
import os
|
||||||
|
|
||||||
PROVISIONER_NAME = os.getenv("PROVISIONER_NAME", "rawfile.csi.openebs.io")
|
PROVISIONER_NAME = os.getenv("PROVISIONER_NAME", "rawfile.csi.openebs.io")
|
||||||
PROVISIONER_VERSION = "0.7.0"
|
PROVISIONER_VERSION = "0.8.0"
|
||||||
DATA_DIR = "/data"
|
DATA_DIR = "/data"
|
||||||
CONFIG = {}
|
CONFIG = {}
|
||||||
RESOURCE_EXHAUSTED_EXIT_CODE = 101
|
RESOURCE_EXHAUSTED_EXIT_CODE = 101
|
||||||
|
|
|
@ -109,6 +109,6 @@ def be_fs_expanded(dev, path):
|
||||||
elif fs == "btrfs":
|
elif fs == "btrfs":
|
||||||
run(f"btrfs filesystem resize max {path}")
|
run(f"btrfs filesystem resize max {path}")
|
||||||
elif fs == "xfs":
|
elif fs == "xfs":
|
||||||
run(f"xfs_growfs -d {dev}")
|
run(f"xfs_growfs -d {path}")
|
||||||
else:
|
else:
|
||||||
raise Exception(f"Unsupported fsType: {fs}")
|
raise Exception(f"Unsupported fsType: {fs}")
|
||||||
|
|
|
@ -2,5 +2,5 @@ apiVersion: v2
|
||||||
name: rawfile-csi
|
name: rawfile-csi
|
||||||
description: RawFile Driver Container Storage Interface
|
description: RawFile Driver Container Storage Interface
|
||||||
type: application
|
type: application
|
||||||
version: 0.7.0
|
version: 0.8.0
|
||||||
appVersion: 0.7.0
|
appVersion: 0.8.0
|
||||||
|
|
|
@ -124,3 +124,34 @@ roleRef:
|
||||||
kind: ClusterRole
|
kind: ClusterRole
|
||||||
name: {{ include "rawfile-csi.fullname" . }}-resizer
|
name: {{ include "rawfile-csi.fullname" . }}-resizer
|
||||||
apiGroup: rbac.authorization.k8s.io
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
---
|
||||||
|
kind: ClusterRole
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["events"]
|
||||||
|
verbs: ["list", "watch", "create", "update", "patch"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshotclasses"]
|
||||||
|
verbs: ["get", "list", "watch"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshotcontents"]
|
||||||
|
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
|
||||||
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||||
|
resources: ["volumesnapshotcontents/status"]
|
||||||
|
verbs: ["update", "patch"]
|
||||||
|
---
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: {{ include "rawfile-csi.fullname" . }}-driver
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: {{ include "rawfile-csi.fullname" . }}-snapshotter
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
|
|
@ -153,3 +153,20 @@ spec:
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: socket-dir
|
- name: socket-dir
|
||||||
mountPath: /csi
|
mountPath: /csi
|
||||||
|
- name: external-snapshotter
|
||||||
|
image: gcr.io/k8s-staging-sig-storage/csi-snapshotter:v5.0.1
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- "--csi-address=$(ADDRESS)"
|
||||||
|
- "--node-deployment=true"
|
||||||
|
- "--extra-create-metadata=true"
|
||||||
|
env:
|
||||||
|
- name: ADDRESS
|
||||||
|
value: /csi/csi.sock
|
||||||
|
- name: NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
volumeMounts:
|
||||||
|
- name: socket-dir
|
||||||
|
mountPath: /csi
|
||||||
|
|
|
@ -3,7 +3,7 @@ provisionerName: "rawfile.csi.openebs.io"
|
||||||
defaults: &defaults
|
defaults: &defaults
|
||||||
image:
|
image:
|
||||||
repository: docker.io/openebs/rawfile-localpv
|
repository: docker.io/openebs/rawfile-localpv
|
||||||
tag: 0.7.0
|
tag: 0.8.0
|
||||||
pullPolicy: Always
|
pullPolicy: Always
|
||||||
resources:
|
resources:
|
||||||
limits:
|
limits:
|
||||||
|
|
|
@ -66,9 +66,9 @@ def run_on_node(fn, node):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
wait_for(is_finished, "task to finish")
|
wait_for(is_finished, "task to finish")
|
||||||
|
task_pod.delete()
|
||||||
if task_pod.obj["status"]["phase"] != "Succeeded":
|
if task_pod.obj["status"]["phase"] != "Succeeded":
|
||||||
exit_code = task_pod.obj["status"]["containerStatuses"][0]["state"][
|
exit_code = task_pod.obj["status"]["containerStatuses"][0]["state"][
|
||||||
"terminated"
|
"terminated"
|
||||||
]["exitCode"]
|
]["exitCode"]
|
||||||
raise CalledProcessError(returncode=exit_code, cmd=f"Task: {name}")
|
raise CalledProcessError(returncode=exit_code, cmd=f"Task: {name}")
|
||||||
task_pod.delete()
|
|
||||||
|
|
|
@ -141,6 +141,7 @@ class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer):
|
||||||
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)),
|
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)),
|
||||||
Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)),
|
Cap(rpc=Cap.RPC(type=Cap.RPC.GET_CAPACITY)),
|
||||||
Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
|
Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
|
||||||
|
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_SNAPSHOT)),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
55
remote.py
55
remote.py
|
@ -1,3 +1,5 @@
|
||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
from util import remote_fn
|
from util import remote_fn
|
||||||
|
|
||||||
|
|
||||||
|
@ -81,3 +83,56 @@ def expand_rawfile(volume_id, size):
|
||||||
{"size": size},
|
{"size": size},
|
||||||
)
|
)
|
||||||
run(f"truncate -s {size} {img_file}")
|
run(f"truncate -s {size} {img_file}")
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def mount_root_subvol(volume_id):
|
||||||
|
import tempfile
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
import rawfile_util
|
||||||
|
from util import run
|
||||||
|
|
||||||
|
root_subvol = tempfile.mkdtemp(prefix="rawfile-")
|
||||||
|
|
||||||
|
img_file = rawfile_util.img_file(volume_id)
|
||||||
|
loop_dev = rawfile_util.attach_loop(img_file)
|
||||||
|
|
||||||
|
run(f"mount -t btrfs -o subvolid=0 {loop_dev} {root_subvol}")
|
||||||
|
try:
|
||||||
|
yield root_subvol
|
||||||
|
finally:
|
||||||
|
run(f"umount {root_subvol}")
|
||||||
|
pathlib.Path(root_subvol).rmdir()
|
||||||
|
|
||||||
|
|
||||||
|
def btrfs_delete_snapshot(volume_id, name):
|
||||||
|
import btrfsutil
|
||||||
|
|
||||||
|
with mount_root_subvol(volume_id) as root_subvol:
|
||||||
|
snapshots_dir = f"{root_subvol}/.snapshots"
|
||||||
|
snapshot_path = f"{snapshots_dir}/{name}"
|
||||||
|
btrfsutil.delete_subvolume(snapshot_path)
|
||||||
|
|
||||||
|
|
||||||
|
def btrfs_create_snapshot(volume_id, name):
|
||||||
|
import btrfsutil
|
||||||
|
import time
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
# TODO: check fstype
|
||||||
|
|
||||||
|
with mount_root_subvol(volume_id) as root_subvol:
|
||||||
|
default_subvol_id = btrfsutil.get_default_subvolume(root_subvol)
|
||||||
|
default_subvol = btrfsutil.subvolume_path(root_subvol, default_subvol_id)
|
||||||
|
default_subvol = f"{root_subvol}/{default_subvol}"
|
||||||
|
|
||||||
|
snapshots_dir = f"{root_subvol}/.snapshots"
|
||||||
|
pathlib.Path(snapshots_dir).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
snapshot_subvol = f"{snapshots_dir}/{name}"
|
||||||
|
btrfsutil.create_snapshot(default_subvol, snapshot_subvol, read_only=True)
|
||||||
|
|
||||||
|
snapshot_id = f"{volume_id}/{name}"
|
||||||
|
creation_time_ns = time.time_ns()
|
||||||
|
return snapshot_id, creation_time_ns
|
||||||
|
|
|
@ -5,3 +5,4 @@ pyyaml
|
||||||
pykube-ng
|
pykube-ng
|
||||||
munch
|
munch
|
||||||
prometheus_client
|
prometheus_client
|
||||||
|
#https://github.com/kdave/btrfs-progs/archive/refs/tags/v5.16.1.tar.gz#egg=btrfsutil&subdirectory=libbtrfsutil/python
|
||||||
|
|
|
@ -1,43 +1,42 @@
|
||||||
#
|
#
|
||||||
# This file is autogenerated by pip-compile with python 3.9
|
# This file is autogenerated by pip-compile with python 3.10
|
||||||
# To update, run:
|
# To update, run:
|
||||||
#
|
#
|
||||||
# pip-compile
|
# pip-compile
|
||||||
#
|
#
|
||||||
certifi==2021.5.30
|
certifi==2021.10.8
|
||||||
# via requests
|
# via requests
|
||||||
chardet==4.0.0
|
charset-normalizer==2.0.12
|
||||||
# via requests
|
# via requests
|
||||||
click==8.0.1
|
click==8.0.3
|
||||||
# via -r requirements.in
|
# via -r requirements.in
|
||||||
grpcio==1.38.1
|
grpcio==1.43.0
|
||||||
# via
|
# via
|
||||||
# -r requirements.in
|
# -r requirements.in
|
||||||
# grpcio-tools
|
# grpcio-tools
|
||||||
grpcio-tools==1.38.1
|
grpcio-tools==1.43.0
|
||||||
# via -r requirements.in
|
# via -r requirements.in
|
||||||
idna==2.10
|
idna==3.3
|
||||||
# via requests
|
# via requests
|
||||||
munch==2.5.0
|
munch==2.5.0
|
||||||
# via -r requirements.in
|
# via -r requirements.in
|
||||||
prometheus-client==0.11.0
|
prometheus-client==0.13.1
|
||||||
# via -r requirements.in
|
# via -r requirements.in
|
||||||
protobuf==3.17.3
|
protobuf==3.19.4
|
||||||
# via grpcio-tools
|
# via grpcio-tools
|
||||||
pykube-ng==21.3.0
|
pykube-ng==22.1.0
|
||||||
# via -r requirements.in
|
# via -r requirements.in
|
||||||
pyyaml==5.4.1
|
pyyaml==6.0
|
||||||
# via
|
# via
|
||||||
# -r requirements.in
|
# -r requirements.in
|
||||||
# pykube-ng
|
# pykube-ng
|
||||||
requests==2.25.1
|
requests==2.27.1
|
||||||
# via pykube-ng
|
# via pykube-ng
|
||||||
six==1.16.0
|
six==1.16.0
|
||||||
# via
|
# via
|
||||||
# grpcio
|
# grpcio
|
||||||
# munch
|
# munch
|
||||||
# protobuf
|
urllib3==1.26.8
|
||||||
urllib3==1.26.6
|
|
||||||
# via requests
|
# via requests
|
||||||
|
|
||||||
# The following packages are considered to be unsafe in a requirements file:
|
# The following packages are considered to be unsafe in a requirements file:
|
||||||
|
|
Loading…
Reference in New Issue