storage: test large volumes

README.md

@@ -175,6 +175,11 @@ Some tests accept an optional `--vm=OVA_URL|VM_key|IP_address` parameter. Those
 If `--vm` is not specified, defaults defined by the tests will be used.
 The `--vm` parameter can be specified several times. Then pytest will run several instances of the tests sequentially, one for each VM.
 
+Some tests support an optional `--volume-size` parameter to configure the size of test VDIs (virtual disk images). This parameter accepts size strings like `1GiB`, `2.5TiB`, `500MiB`, or byte values. The default is `1GiB`. This is useful for testing with larger volumes. Example:
+```
+pytest tests/storage/ext/test_ext_sr.py --hosts=10.0.0.1 --volume-size=10GiB
+```
+
 See also, below: "Markers and test selection" and "Running test jobs with `jobs.py`"
 
 ### Test log level

conftest.py

@@ -18,6 +18,7 @@
     HostAddress,
     callable_marker,
     is_uuid,
+    parse_size,
     prefix_object_name,
     setup_formatted_and_mounted_disk,
     shortened_nodeid,
@@ -102,10 +103,17 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         help="Format of VDI to execute tests on."
         "Example: vhd,qcow2"
     )
+    parser.addoption(
+        "--volume-size",
+        action="store",
+        default="1GiB",
+        help="Default volume size for tests"
+    )
 
 def pytest_configure(config: pytest.Config) -> None:
     global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner')
     global_config.ssh_output_max_lines = int(config.getoption('--ssh-output-max-lines'))
+    global_config.volume_size = parse_size(config.getoption('--volume-size'))
 
 def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
     if "vm_ref" in metafunc.fixturenames:

lib/common.py

@@ -40,6 +40,25 @@
 GiB = KiB**3
 TiB = KiB**4
 
+def parse_size(size_str: str) -> int:
+    """
+    Parse a size string like "2.5TiB", "1GiB" or "1024".
+    """
+    try:
+        return int(size_str)
+    except ValueError:
+        pass
+
+    size_str = size_str.strip()
+    for unit, multiplier in [('TiB', TiB), ('GiB', GiB), ('MiB', MiB), ('KiB', KiB)]:
+        if size_str.endswith(unit):
+            try:
+                return int(float(size_str[:-len(unit)].strip()) * multiplier)
+            except ValueError:
+                pass
+
+    raise ValueError(f"Cannot parse size: {size_str}")
+
 T = TypeVar("T")
 
 HostAddress: TypeAlias = str

lib/config.py

@@ -1,7 +1,8 @@
-
+from lib.common import GiB
 
 ignore_ssh_banner = False
 ssh_output_max_lines = 20
+volume_size = 1 * GiB
 
 def sr_device_config(datakey: str, *, required: list[str] = []) -> dict[str, str]:
     import data  # import here to avoid depending on this user file for collecting tests

tests/storage/__init__.py

@@ -6,6 +6,7 @@
     cold_migration_then_come_back,
     install_randstream,
     live_storage_migration_then_come_back,
+    randstream,
     try_to_create_sr_with_missing_device,
     vdi_export_import,
     vdi_is_open,

tests/storage/cephfs/conftest.py

@@ -43,8 +43,8 @@ def cephfs_sr(host: Host, cephfs_device_config: dict[str, str], pool_with_ceph:
     sr.destroy()
 
 @pytest.fixture(scope='module')
-def vdi_on_cephfs_sr(cephfs_sr: SR) -> Generator[VDI, None, None]:
-    vdi = cephfs_sr.create_vdi('CephFS-VDI-test')
+def vdi_on_cephfs_sr(cephfs_sr) -> Generator[VDI, None, None]:
+    vdi = cephfs_sr.create_vdi('CephFS-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/conftest.py

@@ -2,6 +2,11 @@
 
 import pytest
 
+import logging
+
+from lib import config
+from lib.common import randid
+from lib.host import Host
 from lib.vm import VM
 from tests.storage import install_randstream
 
@@ -19,3 +24,25 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item
 def storage_test_vm(running_unix_vm: VM) -> Generator[VM, None, None]:
     install_randstream(running_unix_vm)
     yield running_unix_vm
+
+@pytest.fixture()
+def temp_large_dir(host: Host) -> Generator[str, None, None]:
+    """Create a temporary directory on a large (NFS) disk on the primary host"""
+    temp_id = randid()
+    nfs_conf = config.sr_device_config("NFS_DEVICE_CONFIG")
+    if 'serverpath' in nfs_conf or 'server' in nfs_conf:
+        mount_path = f'/mnt/nfs_{temp_id}'
+        host.ssh(f'mkdir -p {mount_path}')
+        path = f'{mount_path}/temp_large_dir_{temp_id}'
+        host.ssh(f'mount -t nfs {nfs_conf["server"]}:{nfs_conf["serverpath"]} {mount_path}')
+        host.ssh(f'mkdir {path}')
+        yield path
+        host.ssh(f'rm -rf {path}')
+        host.ssh(f'umount {mount_path}')
+        host.ssh(f'rmdir {mount_path}')
+    else:
+        logging.warning("No NFS configuration found, using /tmp")
+        path = f'/tmp/temp_large_dir_{temp_id}'
+        host.ssh(f'mkdir {path}')
+        yield path
+        host.ssh(f'rm -rf {path}')

tests/storage/ext/conftest.py

@@ -4,6 +4,7 @@
 
 import logging
 
+from lib import config
 from lib.host import Host
 from lib.sr import SR
 from lib.vdi import VDI, ImageFormat
@@ -27,7 +28,7 @@ def ext_sr(host: Host,
 
 @pytest.fixture(scope='module')
 def vdi_on_ext_sr(ext_sr: SR) -> Generator[VDI, None, None]:
-    vdi = ext_sr.create_vdi('EXT-local-VDI-test')
+    vdi = ext_sr.create_vdi('EXT-local-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/ext/test_ext_sr.py

@@ -92,12 +92,14 @@ def test_coalesce(self, storage_test_vm: VM, vdi_on_ext_sr: VDI, vdi_op: Coalesc
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, vm_on_ext_sr: VM, compression: XVACompression, defer: Defer) -> None:
-        xva_export_import(vm_on_ext_sr, compression, defer)
+    def test_xva_export_import(self, vm_on_ext_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer) \
+            -> None:
+        xva_export_import(vm_on_ext_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
-    def test_vdi_export_import(self, storage_test_vm: VM, ext_sr: SR, image_format: ImageFormat, defer: Defer) -> None:
-        vdi_export_import(storage_test_vm, ext_sr, image_format, defer)
+    def test_vdi_export_import(self, storage_test_vm: VM, ext_sr: SR, image_format: ImageFormat, temp_large_dir: str,
+                               defer: Defer) -> None:
+        vdi_export_import(storage_test_vm, ext_sr, image_format, temp_large_dir, defer)
 
     # *** tests with reboots (longer tests).
 

tests/storage/glusterfs/conftest.py

@@ -5,6 +5,7 @@
 import logging
 from dataclasses import dataclass
 
+from lib import config
 from lib.common import exec_nofail, raise_errors, setup_formatted_and_mounted_disk, teardown_formatted_and_mounted_disk
 from lib.netutil import is_ipv6
 
@@ -263,7 +264,7 @@ def glusterfs_sr(
 
 @pytest.fixture(scope='module')
 def vdi_on_glusterfs_sr(glusterfs_sr: SR) -> Generator[VDI, None, None]:
-    vdi = glusterfs_sr.create_vdi('GlusterFS-VDI-test')
+    vdi = glusterfs_sr.create_vdi('GlusterFS-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/largeblock/conftest.py

@@ -4,6 +4,7 @@
 
 import logging
 
+from lib import config
 from lib.vdi import ImageFormat
 
 from typing import TYPE_CHECKING, Generator
@@ -28,8 +29,8 @@ def largeblock_sr(host: Host,
     sr.destroy()
 
 @pytest.fixture(scope='module')
-def vdi_on_largeblock_sr(largeblock_sr: SR) -> Generator[VDI, None, None]:
-    vdi = largeblock_sr.create_vdi('LARGEBLOCK-local-VDI-test')
+def vdi_on_largeblock_sr(largeblock_sr) -> Generator[VDI, None, None]:
+    vdi = largeblock_sr.create_vdi('LARGEBLOCK-local-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/linstor/conftest.py

@@ -8,6 +8,7 @@
 from dataclasses import dataclass
 
 import lib.commands as commands
+from lib import config
 
 # explicit import for package-scope fixtures
 from pkgfixtures import pool_with_saved_yum_state
@@ -163,7 +164,7 @@ def linstor_sr(
 
 @pytest.fixture(scope='module')
 def vdi_on_linstor_sr(linstor_sr: SR) -> Generator[VDI, None, None]:
-    vdi = linstor_sr.create_vdi('LINSTOR-VDI-test')
+    vdi = linstor_sr.create_vdi('LINSTOR-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/lvm/conftest.py

@@ -4,6 +4,7 @@
 
 import logging
 
+from lib import config
 from lib.host import Host
 from lib.sr import SR
 from lib.vdi import VDI, ImageFormat
@@ -25,9 +26,9 @@ def lvm_sr(host: Host,
     # teardown
     sr.destroy()
 
-@pytest.fixture(scope='module')
+@pytest.fixture()
 def vdi_on_lvm_sr(lvm_sr: SR) -> Generator[VDI, None, None]:
-    vdi = lvm_sr.create_vdi('LVM-local-VDI-test')
+    vdi = lvm_sr.create_vdi('LVM-local-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/lvm/test_lvm_sr.py

@@ -149,12 +149,14 @@ def test_coalesce(self, storage_test_vm: VM, vdi_on_lvm_sr: VDI, vdi_op: Coalesc
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, vm_on_lvm_sr: VM, compression: XVACompression, defer: Defer) -> None:
-        xva_export_import(vm_on_lvm_sr, compression, defer)
+    def test_xva_export_import(self, vm_on_lvm_sr: VM, compression: XVACompression, temp_large_dir: str, defer: Defer) \
+            -> None:
+        xva_export_import(vm_on_lvm_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
-    def test_vdi_export_import(self, storage_test_vm: VM, lvm_sr: SR, image_format: ImageFormat, defer: Defer) -> None:
-        vdi_export_import(storage_test_vm, lvm_sr, image_format, defer)
+    def test_vdi_export_import(self, storage_test_vm: VM, lvm_sr: SR, image_format: ImageFormat, temp_large_dir: str,
+                               defer: Defer) -> None:
+        vdi_export_import(storage_test_vm, lvm_sr, image_format, temp_large_dir, defer)
 
     # *** tests with reboots (longer tests).
 

tests/storage/lvmoiscsi/conftest.py

@@ -24,9 +24,9 @@ def lvmoiscsi_sr(host: Host, lvmoiscsi_device_config: dict[str, str], image_form
     # teardown
     sr.destroy()
 
-@pytest.fixture(scope='module')
+@pytest.fixture()
 def vdi_on_lvmoiscsi_sr(lvmoiscsi_sr: SR) -> Generator[VDI, None, None]:
-    vdi = lvmoiscsi_sr.create_vdi('lvmoiscsi-VDI-test')
+    vdi = lvmoiscsi_sr.create_vdi('lvmoiscsi-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/lvmoiscsi/test_lvmoiscsi_sr.py

@@ -75,13 +75,14 @@ def test_coalesce(self, storage_test_vm: 'VM', vdi_on_lvmoiscsi_sr: 'VDI', vdi_o
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, vm_on_lvmoiscsi_sr: VM, compression: XVACompression, defer: Defer) -> None:
-        xva_export_import(vm_on_lvmoiscsi_sr, compression, defer)
+    def test_xva_export_import(self, vm_on_lvmoiscsi_sr: VM, compression: XVACompression, temp_large_dir: str,
+                               defer: Defer) -> None:
+        xva_export_import(vm_on_lvmoiscsi_sr, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
-    def test_vdi_export_import(self, storage_test_vm: VM, lvmoiscsi_sr: SR, image_format: ImageFormat, defer: Defer) \
-            -> None:
-        vdi_export_import(storage_test_vm, lvmoiscsi_sr, image_format, defer)
+    def test_vdi_export_import(self, storage_test_vm: VM, lvmoiscsi_sr: SR, image_format: ImageFormat,
+                               temp_large_dir: str, defer: Defer) -> None:
+        vdi_export_import(storage_test_vm, lvmoiscsi_sr, image_format, temp_large_dir, defer)
 
     # *** tests with reboots (longer tests).
 

tests/storage/moosefs/conftest.py

@@ -59,8 +59,8 @@ def moosefs_sr(moosefs_device_config: dict[str, str], pool_with_moosefs_enabled:
     sr.destroy()
 
 @pytest.fixture(scope='module')
-def vdi_on_moosefs_sr(moosefs_sr: SR) -> Generator[VDI, None, None]:
-    vdi = moosefs_sr.create_vdi('MooseFS-VDI-test')
+def vdi_on_moosefs_sr(moosefs_sr) -> Generator[VDI, None, None]:
+    vdi = moosefs_sr.create_vdi('MooseFS-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/nfs/conftest.py

@@ -36,7 +36,7 @@ def nfs_sr(host: Host, image_format: ImageFormat, nfs_device_config: dict[str, s
 
 @pytest.fixture(scope='module')
 def vdi_on_nfs_sr(nfs_sr: SR) -> Generator[VDI, None, None]:
-    vdi = nfs_sr.create_vdi('NFS-VDI-test')
+    vdi = nfs_sr.create_vdi('NFS-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 
@@ -66,7 +66,7 @@ def nfs4_sr(host: Host, image_format: ImageFormat, nfs4_device_config: dict[str,
 
 @pytest.fixture(scope='module')
 def vdi_on_nfs4_sr(nfs4_sr: SR) -> Generator[VDI, None, None]:
-    vdi = nfs4_sr.create_vdi('NFS4-VDI-test')
+    vdi = nfs4_sr.create_vdi('NFS4-VDI-test', virtual_size=config.volume_size)
     yield vdi
     vdi.destroy()
 

tests/storage/nfs/test_nfs_sr.py

@@ -124,14 +124,15 @@ def test_coalesce(self, storage_test_vm: VM, dispatch_nfs: VDI, vdi_op: Coalesce
     @pytest.mark.usefixtures('vm_ref')
     @pytest.mark.parametrize('dispatch_nfs', ['vm_on_nfs_sr', 'vm_on_nfs4_sr'], indirect=True)
     @pytest.mark.parametrize("compression", ["none", "gzip", "zstd"])
-    def test_xva_export_import(self, dispatch_nfs: VM, compression: XVACompression, defer: Defer) -> None:
-        xva_export_import(dispatch_nfs, compression, defer)
+    def test_xva_export_import(self, dispatch_nfs: VM, compression: XVACompression, temp_large_dir: str, defer: Defer) \
+            -> None:
+        xva_export_import(dispatch_nfs, compression, temp_large_dir, defer)
 
     @pytest.mark.small_vm
     @pytest.mark.parametrize('dispatch_nfs', ['nfs_sr', 'nfs4_sr'], indirect=True)
-    def test_vdi_export_import(self, storage_test_vm: VM, dispatch_nfs: SR, image_format: ImageFormat, defer: Defer) \
-            -> None:
-        vdi_export_import(storage_test_vm, dispatch_nfs, image_format, defer)
+    def test_vdi_export_import(self, storage_test_vm: VM, dispatch_nfs: SR, image_format: ImageFormat,
+                               temp_large_dir: str, defer: Defer) -> None:
+        vdi_export_import(storage_test_vm, dispatch_nfs, image_format, temp_large_dir, defer)
 
     # *** tests with reboots (longer tests).