k-space/rawfile-localpv
10
0
Fork 0
Code Issues Pull Requests Packages Projects Activity

Support online volume expansion

Browse Source 
Summary:
Online volume expansion is a 2 phase process:

1. The backing storage, in this case the raw file, needs to be resized. (i.e. `truncate -s`)
2. The node should be notified, so that it can both refresh its device capacity (i.e. `losetup -c`) and resize the filesystem (`resize2fs`) accordingly.

Although in our case both steps could be performed on the node itself, for the sake of following the semantics of how volume expansion works, we perform step 1 from the controller, and step 2 from the node.

Also, the `external-resizer` component is added which watches for PVC size updates, and notifies the CSI controller about it.

Test Plan:
Setup:
- Deploy
- Create a rawfile-backed pvc, and attach a Deployment to it
- Keep an eye on `rawfile` pod logs in `kube-system` namespace to see if any errors pop out during all scenarios

Scenario 1:
- Increase the size of the pvc
- Exec into the pod and verify that the volume is resized indeed (using `df`)

Scenario 2:
- Decrease deployment's replica to 0
- Increase the size of the pvc. Wait for a couple of minutes.
- Increase deployment's replica to 1
- Exec into the pod and verify that the volume is resized indeed.

Reviewers: bghadiri, mhyousefi, h.marvi, sina_rad

Reviewed By: bghadiri, mhyousefi, sina_rad

Differential Revision: https://phab.hamravesh.ir/D817
This commit is contained in:
Mehran Kholdi 2020-06-12 16:42:49 +04:30
parent be2cd1b72c
commit d1c0d49cf0
6 changed files with 102 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -20,6 +20,7 @@ metadata:
provisioner: rawfile.hamravesh.com
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true
```
Features
@ -38,7 +39,7 @@ Features
- [ ] `Block` mode
- [x] Volume metrics
- [ ] Supports fsTypes
- [ ] Online expansion: If fs supports it (e.g. ext4, btrfs)
- [x] Online expansion: If fs supports it (e.g. ext4, btrfs)
- [ ] Online shrinking: If fs supports it (e.g. btrfs)
- [ ] Offline expansion/shrinking
- [ ] Ephemeral inline volume

2
deploy/charts/rawfile-csi/Chart.yaml
View File

@ -2,5 +2,5 @@ apiVersion: v2
name: rawfile-csi
description: RawFile Driver Container Storage Interface
type: application
version: 0.1.3
version: 0.1.4
appVersion: 0.0.1

37
deploy/charts/rawfile-csi/templates/00-rbac.yaml
View File

@ -75,3 +75,40 @@ roleRef:
kind: ClusterRole
name: {{ include "rawfile-csi.fullname" . }}-broker
apiGroup: rbac.authorization.k8s.io
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "rawfile-csi.fullname" . }}-resizer
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
verbs: ["patch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "rawfile-csi.fullname" . }}-resizer
subjects:
- kind: ServiceAccount
name: {{ include "rawfile-csi.fullname" . }}-driver
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ include "rawfile-csi.fullname" . }}-resizer
apiGroup: rbac.authorization.k8s.io

11
deploy/charts/rawfile-csi/templates/01-controller-plugin.yaml
View File

@ -64,3 +64,14 @@ spec:
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-resizer
image: quay.io/k8scsi/csi-resizer:v0.5.0
imagePullPolicy: IfNotPresent
args:
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /csi/csi.sock
volumeMounts:
- name: socket-dir
mountPath: /csi

42
rawfile_servicer.py
View File

@ -7,7 +7,7 @@ import rawfile_util
from csi import csi_pb2, csi_pb2_grpc
from orchestrator.k8s import volume_to_node, run_on_node
from rawfile_util import attach_loop, detach_loops
from remote import init_rawfile, scrub
from remote import init_rawfile, scrub, expand_rawfile
from util import log_grpc_request, run
NODE_NAME_TOPOLOGY_KEY = "hostname"
@ -31,6 +31,11 @@ class RawFileIdentityServicer(csi_pb2_grpc.IdentityServicer):
type=Cap.Service.VOLUME_ACCESSIBILITY_CONSTRAINTS
)
),
Cap(
volume_expansion=Cap.VolumeExpansion(
type=Cap.VolumeExpansion.ONLINE
)
),
]
)
@ -47,7 +52,10 @@ class RawFileNodeServicer(csi_pb2_grpc.NodeServicer):
def NodeGetCapabilities(self, request, context):
Cap = csi_pb2.NodeServiceCapability
return csi_pb2.NodeGetCapabilitiesResponse(
capabilities=[Cap(rpc=Cap.RPC(type=Cap.RPC.STAGE_UNSTAGE_VOLUME))]
capabilities=[
Cap(rpc=Cap.RPC(type=Cap.RPC.STAGE_UNSTAGE_VOLUME)),
Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
]
)
@log_grpc_request
@ -100,13 +108,28 @@ class RawFileNodeServicer(csi_pb2_grpc.NodeServicer):
detach_loops(img_file)
return csi_pb2.NodeUnstageVolumeResponse()
@log_grpc_request
def NodeExpandVolume(self, request, context):
volume_id = request.volume_id
size = request.capacity_range.required_bytes
img_file = rawfile_util.img_file(volume_id)
for dev in rawfile_util.attached_loops(img_file):
run(f"losetup -c {dev}")
if True: # TODO: is ext2/ext3/ext4
run(f"resize2fs {dev}")
break
return csi_pb2.NodeExpandVolumeResponse(capacity_bytes=size)
class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer):
@log_grpc_request
def ControllerGetCapabilities(self, request, context):
Cap = csi_pb2.ControllerServiceCapability
return csi_pb2.ControllerGetCapabilitiesResponse(
capabilities=[Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME))]
capabilities=[
Cap(rpc=Cap.RPC(type=Cap.RPC.CREATE_DELETE_VOLUME)),
Cap(rpc=Cap.RPC(type=Cap.RPC.EXPAND_VOLUME)),
]
)
@log_grpc_request
@ -177,3 +200,16 @@ class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer):
node_name = volume_to_node(request.volume_id)
run_on_node(scrub.as_cmd(volume_id=request.volume_id), node=node_name)
return csi_pb2.DeleteVolumeResponse()
@log_grpc_request
def ControllerExpandVolume(self, request, context):
volume_id = request.volume_id
node_name = volume_to_node(volume_id)
size = request.capacity_range.required_bytes
run_on_node(
expand_rawfile.as_cmd(volume_id=volume_id, size=size), node=node_name
)
return csi_pb2.ControllerExpandVolumeResponse(
capacity_bytes=size, node_expansion_required=True,
)

12
remote.py
View File

@ -29,3 +29,15 @@ def init_rawfile(volume_id, size):
)
run(f"truncate -s {size} {img_file}")
run(f"mkfs.ext4 {img_file}")
@remote_fn
def expand_rawfile(volume_id, size):
import rawfile_util
from util import run
img_file = rawfile_util.img_file(volume_id)
rawfile_util.patch_metadata(
volume_id, {"size": size},
)
run(f"truncate -s {size} {img_file}")