From b2c4b779112fd85c6ac00cbef7ceffa210cd383e Mon Sep 17 00:00:00 2001
From: Mehran Kholdi <semekh@hamravesh.com>
Date: Fri, 29 May 2020 20:23:26 +0430
Subject: [PATCH] Implement `STAGE_UNSTAGE_VOLUME` capability

Summary: Before this, we directly mounted the the rawfile on the mountpoint. In this revision the `STAGE_UNSTAGE_VOLUME` capability is implemented, meaning that the volume is first mounted to a staging path, and then `bind`-mounted to the actual path. This way we can free up loopback devices when they are not needed.

Test Plan:
- Create a pvc, and use it inside a pod
- Run `losetup -l` on the node running the pod, and assert the creation of a loop device
- Delete the pod, but not the pvc
- Run `losetup -l` on the same node, and assert the removal of the loop device

Reviewers: h.marvi, bghadiri

Differential Revision: https://phab.hamravesh.ir/D806
---
 deploy/charts/rawfile-csi/Chart.yaml          |  2 +-
 .../rawfile-csi/templates/01-node-plugin.yaml |  4 +-
 rawfile_servicer.py                           | 42 ++++++++++++++++---
 rawfile_util.py                               |  6 +++
 4 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/deploy/charts/rawfile-csi/Chart.yaml b/deploy/charts/rawfile-csi/Chart.yaml
index e9cc096..9d8ff24 100644
--- a/deploy/charts/rawfile-csi/Chart.yaml
+++ b/deploy/charts/rawfile-csi/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: rawfile-csi
 description: RawFile Driver Container Storage Interface
 type: application
-version: 0.1.2
+version: 0.1.3
 appVersion: 0.0.1
diff --git a/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml b/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml
index 0159d88..3c4d232 100644
--- a/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml
+++ b/deploy/charts/rawfile-csi/templates/01-node-plugin.yaml
@@ -47,7 +47,7 @@ spec:
             type: DirectoryOrCreate
         - name: mountpoint-dir
           hostPath:
-            path: /var/lib/kubelet/pods
+            path: /var/lib/kubelet
             type: DirectoryOrCreate
         - name: data-dir
           hostPath:
@@ -76,7 +76,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
             - name: mountpoint-dir
-              mountPath: /var/lib/kubelet/pods
+              mountPath: /var/lib/kubelet
               mountPropagation: "Bidirectional"
             - name: data-dir
               mountPath: /data
diff --git a/rawfile_servicer.py b/rawfile_servicer.py
index bb2d921..b5e0e02 100644
--- a/rawfile_servicer.py
+++ b/rawfile_servicer.py
@@ -1,10 +1,12 @@
+from pathlib import Path
+
 import grpc
 from google.protobuf.wrappers_pb2 import BoolValue
 
 import rawfile_util
 from csi import csi_pb2, csi_pb2_grpc
 from orchestrator.k8s import volume_to_node, run_on_node
-from rawfile_util import attach_loop
+from rawfile_util import attach_loop, detach_loops
 from remote import init_rawfile, scrub
 from util import log_grpc_request, run
 
@@ -43,14 +45,16 @@ class RawFileNodeServicer(csi_pb2_grpc.NodeServicer):
 
     @log_grpc_request
     def NodeGetCapabilities(self, request, context):
-        return csi_pb2.NodeGetCapabilitiesResponse(capabilities=[])
+        Cap = csi_pb2.NodeServiceCapability
+        return csi_pb2.NodeGetCapabilitiesResponse(
+            capabilities=[Cap(rpc=Cap.RPC(type=Cap.RPC.STAGE_UNSTAGE_VOLUME))]
+        )
 
     @log_grpc_request
     def NodePublishVolume(self, request, context):
         mount_path = request.target_path
-        img_file = rawfile_util.img_file(request.volume_id)
-        loop_file = attach_loop(img_file)
-        run(f"mount {loop_file} {mount_path}")
+        staging_path = request.staging_target_path
+        run(f"mount --bind {staging_path}/mount {mount_path}")
         return csi_pb2.NodePublishVolumeResponse()
 
     @log_grpc_request
@@ -68,6 +72,34 @@ class RawFileNodeServicer(csi_pb2_grpc.NodeServicer):
             ),
         )
 
+    @log_grpc_request
+    def NodeStageVolume(self, request, context):
+        img_file = rawfile_util.img_file(request.volume_id)
+        loop_file = attach_loop(img_file)
+        staging_path = request.staging_target_path
+        device_path = Path(f"{staging_path}/device")
+        if not device_path.exists():
+            device_path.symlink_to(loop_file)
+        mount_path = Path(f"{staging_path}/mount")
+        if not mount_path.exists():
+            mount_path.mkdir()
+            run(f"mount {device_path} {mount_path}")
+        return csi_pb2.NodeStageVolumeResponse()
+
+    @log_grpc_request
+    def NodeUnstageVolume(self, request, context):
+        img_file = rawfile_util.img_file(request.volume_id)
+        staging_path = request.staging_target_path
+        mount_path = Path(f"{staging_path}/mount")
+        if mount_path.exists():
+            run(f"umount {mount_path}")
+            mount_path.rmdir()
+        device_path = Path(f"{staging_path}/device")
+        if device_path.exists():
+            device_path.unlink()
+        detach_loops(img_file)
+        return csi_pb2.NodeUnstageVolumeResponse()
+
 
 class RawFileControllerServicer(csi_pb2_grpc.ControllerServicer):
     @log_grpc_request
diff --git a/rawfile_util.py b/rawfile_util.py
index 6376a19..ff66220 100644
--- a/rawfile_util.py
+++ b/rawfile_util.py
@@ -57,6 +57,12 @@ def attach_loop(file) -> str:
         run(f"losetup --direct-io=on -f {file}")
 
 
+def detach_loops(file) -> None:
+    devs = attached_loops(file)
+    for dev in devs:
+        run(f"losetup -d {dev}")
+
+
 def list_all_volumes():
     metas = glob.glob(f"{DATA_DIR}/*/disk.meta")
     return [basename(dirname(meta)) for meta in metas]