feat(coalesce): Add tests for coalesce

Signed-off-by: Damien Thenot <[email protected]>

Author: Nambrok

lib/host.py

@@ -4,6 +4,7 @@
 import shlex
 import tempfile
 import uuid
+from typing import Optional
 
 from packaging import version
 
@@ -606,3 +607,16 @@ def enable_hsts_header(self):
     def disable_hsts_header(self):
         self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf'])
         self.restart_toolstack(verify=True)
+
+    def get_dom0_uuid(self):
+        output = self.ssh(["grep", "-e", "\"CONTROL_DOMAIN_UUID=\"", "/etc/xensource-inventory"])
+        return output.split("=")[1].replace("'", "")
+
+    def get_vdi_sr_uuid(self, vdi_uuid) -> Optional[SR]:
+        sr_uuid = self.xe("vdi-param-get",
+                          {"param-name": "sr-uuid",
+                           "uuid": vdi_uuid,
+                           })
+        if sr_uuid is None:
+            return None
+        return SR(sr_uuid, self.pool)

lib/vdi.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set
 
@@ -9,11 +10,7 @@ def __init__(self, uuid, *, host=None, sr=None):
         self.uuid = uuid
         # TODO: use a different approach when migration is possible
         if sr is None:
-            sr_uuid = host.get_vdi_sr_uuid(uuid)
-            # avoid circular import
-            # FIXME should get it from Host instead
-            from lib.sr import SR
-            self.sr = SR(sr_uuid, host.pool)
+            self.sr = host.get_vdi_sr_uuid(uuid)
         else:
             self.sr = sr
 
@@ -34,6 +31,9 @@ def readonly(self):
     def __str__(self):
         return f"VDI {self.uuid} on SR {self.sr.uuid}"
 
+    def get_parent(self) -> Optional[str]:
+        return self.param_get("sm-config", key="vhd-parent", accept_unknown_key=True)
+
     def param_get(self, param_name, key=None, accept_unknown_key=False):
         return _param_get(self.sr.pool.master, self.xe_prefix, self.uuid,
                           param_name, key, accept_unknown_key)

tests/storage/coalesce/conftest.py

@@ -0,0 +1,66 @@
+import pytest
+import logging
+
+from lib.vdi import VDI
+
+MAX_LENGTH = 1 * 1024 * 1024 * 1024 # 1GiB
+
+@pytest.fixture(scope="package", params=["qcow2", "vhd"])
+def vdi_type(request):
+    yield request.param
+
+@pytest.fixture(scope="module")
+def vdi_on_local_sr(host, local_sr_on_hostA1, vdi_type):
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = host.xe("vdi-create",
+                       {"sr-uuid": sr_uuid,
+                        "name-label": "testVDI",
+                        "virtual-size": str(MAX_LENGTH),
+                        "sm-config:type": vdi_type,
+                        })
+    logging.info(">> Created VDI {} of type {}".format(vdi_uuid, vdi_type))
+
+    vdi = VDI(vdi_uuid, host=host)
+
+    yield vdi
+
+    logging.info("<< Destroying VDI {}".format(vdi_uuid))
+    host.xe("vdi-destroy", {"uuid": vdi_uuid})
+
+@pytest.fixture(scope="module")
+def vdi_with_vbd_on_dom0(host, vdi_on_local_sr):
+    vdi_uuid = vdi_on_local_sr.uuid
+    dom0_uuid = host.get_dom0_uuid()
+    logging.info(f">> Plugging VDI {vdi_uuid} on Dom0")
+    vbd_uuid = host.xe("vbd-create",
+                       {"vdi-uuid": vdi_uuid,
+                        "vm-uuid": dom0_uuid,
+                        "device": "autodetect",
+                        })
+    host.xe("vbd-plug", {"uuid": vbd_uuid})
+
+    yield vdi_on_local_sr
+
+    logging.info(f"<< Unplugging VDI {vdi_uuid} from Dom0")
+    host.xe("vbd-unplug", {"uuid": vbd_uuid})
+    host.xe("vbd-destroy", {"uuid": vbd_uuid})
+
+@pytest.fixture(scope="class")
+def data_file_on_host(host):
+    filename = "/root/data.bin"
+    logging.info(f">> Creating data file {filename} on host")
+    size = 1 * 1024 * 1024 # 1MiB
+    assert size <= MAX_LENGTH, "Size of the data file bigger than the VDI size"
+
+    host.ssh(["dd", "if=/dev/urandom", f"of={filename}", f"bs={size}", "count=1"])
+
+    yield filename
+
+    logging.info("<< Deleting data file")
+    host.ssh(["rm", filename])
+
+@pytest.fixture(scope="module")
+def tapdev(local_sr_on_hostA1, vdi_with_vbd_on_dom0):
+    sr_uuid = local_sr_on_hostA1.uuid
+    vdi_uuid = vdi_with_vbd_on_dom0.uuid
+    yield f"/dev/sm/backend/{sr_uuid}/{vdi_uuid}"

tests/storage/coalesce/test_coalesce.py

@@ -0,0 +1,93 @@
+import logging
+from typing import Optional
+import hashlib
+
+import time
+
+from lib.host import Host
+
+def copy_data_to_tapdev(host: Host, data_file: str, tapdev: str, offset: int, length: int):
+    """
+    if offset == 0:
+        off = "0"
+    else:
+        off = f"{offset}B" # Doesn't work with `dd` version of XCP-ng 8.3
+    """
+    bs = 1
+    off = int(offset / bs)
+    count = length / bs
+    count += length % bs
+    count = int(count)
+    cmd = ["dd", f"if={data_file}", f"of={tapdev}", f"bs={bs}", f"seek={off}", f"count={count}"]
+    host.ssh(cmd)
+
+def get_data(host: Host, file: str, offset: int, length: int):
+    return host.ssh(["xxd", "-p", "-seek", str(offset), "-len", str(length), file])
+
+def get_hashed_data(host: Host, file: str, offset: int, length: int):
+    return hashlib.md5(get_data(host, file, offset, length).encode()).hexdigest()
+
+def snapshot_vdi(host: Host, vdi_uuid: str):
+    vdi_snap = host.xe("vdi-snapshot", {"uuid": vdi_uuid})
+    logging.info(f"Snapshot VDI {vdi_uuid}: {vdi_snap}")
+    return vdi_snap
+
+def compare_data(host: Host, tapdev: str, data_file: str, offset: int, length: int) -> bool:
+    logging.info("Getting data from VDI and file")
+    vdi_checksum = get_hashed_data(host, tapdev, offset, length)
+    file_checksum = get_hashed_data(host, data_file, 0, length)
+    logging.info(f"VDI: {vdi_checksum}")
+    logging.info(f"FILE: {file_checksum}")
+
+    return vdi_checksum == file_checksum
+
+def test_write_data(host, tapdev, data_file_on_host):
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+def test_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host):
+    vdi = vdi_with_vbd_on_dom0
+    vdi_uuid = vdi.uuid
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    vdi_snap = snapshot_vdi(host, vdi_uuid)
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    logging.info("Removing VDI snapshot")
+    host.xe("vdi-destroy", {"uuid": vdi_snap})
+
+    logging.info("Waiting for coalesce")
+    while vdi.get_parent() is not None:
+        time.sleep(1)
+    logging.info("Coalesce done")
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)
+
+def test_clone_coalesce(host, tapdev, vdi_with_vbd_on_dom0, data_file_on_host):
+    vdi = vdi_with_vbd_on_dom0
+    vdi_uuid = vdi.uuid
+    length = 1 * 1024 * 1024
+    offset = 0
+
+    clone_uuid = host.xe("vdi-clone", {"uuid": vdi_uuid})
+    logging.info(f"Clone VDI {vdi_uuid}: {clone_uuid}")
+
+    logging.info("Copying data to tapdev")
+    copy_data_to_tapdev(host, data_file_on_host, tapdev, offset, length)
+
+    host.xe("vdi-destroy", {"uuid": clone_uuid})
+
+    logging.info("Waiting for coalesce")
+    while vdi.get_parent() is not None:
+        time.sleep(1)
+    logging.info("Coalesce done")
+
+    assert compare_data(host, tapdev, data_file_on_host, offset, length)