diff --git a/.github/workflows/tests-gpu.yml b/.github/workflows/tests-gpu.yml index 40f1dacf..a0b49353 100644 --- a/.github/workflows/tests-gpu.yml +++ b/.github/workflows/tests-gpu.yml @@ -9,6 +9,7 @@ jobs: name: GPU Tests runs-on: labels: int-linux-x64-4core-gpu-t4-ubuntu24.04-1 + permissions: contents: read @@ -43,4 +44,4 @@ jobs: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Tests - run: poetry run -- pytest -rA --setup-show test/integration/gpu + run: poetry run -- pytest -rA --setup-show test/integration/gpu/ diff --git a/.github/workflows/tests-ordinary-integration.yml b/.github/workflows/tests-ordinary-integration.yml index 786dab37..1dfd1f2a 100644 --- a/.github/workflows/tests-ordinary-integration.yml +++ b/.github/workflows/tests-ordinary-integration.yml @@ -13,16 +13,29 @@ jobs: strategy: fail-fast: false matrix: - file: - - test_itde_manager.py - - test_sagemaker_extension_wrapper.py - - itest_slc.py - - test_transformers_extension_wrapper.py - - itde_mgr_in_container/test_itde_connect.py - - itde_mgr_in_container/test_itde_external.py - - itde_mgr_in_container/test_itde_recreation_after_take_down.py - - itde_mgr_in_container/test_itde_recreation_without_take_down.py - - itde_mgr_in_container/test_itde_stop_and_restart.py + include: + - file: test_itde_manager.py + extra_param: "" + - file: test_sagemaker_extension_wrapper.py + extra_param: "" + - file: test_transformers_extension_wrapper.py + extra_param: "" + - file: itde_mgr_in_container/test_itde_connect.py + extra_param: "" + - file: itde_mgr_in_container/test_itde_external.py + extra_param: "" + - file: itde_mgr_in_container/test_itde_recreation_after_take_down.py + extra_param: "" + - file: itde_mgr_in_container/test_itde_recreation_without_take_down.py + extra_param: "" + - file: itde_mgr_in_container/test_itde_stop_and_restart.py + extra_param: "" + - file: itest_slc.py + extra_param: "--package-manager pip --compression-strategy gzip" + - file: itest_slc.py + extra_param: "--package-manager conda --compression-strategy gzip" + - file: itest_slc.py + extra_param: "--package-manager pip --compression-strategy none" steps: @@ -53,4 +66,4 @@ jobs: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Pytest - run: poetry run -- pytest -rA --setup-show test/integration/ordinary/${{ matrix.file }} + run: poetry run -- pytest -rA --setup-show test/integration/ordinary/${{ matrix.file }} ${{matrix.extra_param}} diff --git a/.gitignore b/.gitignore index 5bdeed0a..7659f67e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ __pycache__/ .build_output # drawio backup files /doc/user_guide/*.bkp +.workspace \ No newline at end of file diff --git a/exasol/nb_connector/slc/__init__.py b/exasol/nb_connector/slc/__init__.py index cd6b431f..5070661b 100644 --- a/exasol/nb_connector/slc/__init__.py +++ b/exasol/nb_connector/slc/__init__.py @@ -1,5 +1,6 @@ from exasol.nb_connector.slc.script_language_container import ( + CondaPackageDefinition, PipPackageDefinition, ScriptLanguageContainer, ) -from exasol.nb_connector.slc.slc_flavor import SlcError +from exasol.nb_connector.slc.slc_error import SlcError diff --git a/exasol/nb_connector/slc/script_language_container.py b/exasol/nb_connector/slc/script_language_container.py index 45485ba3..98669e60 100644 --- a/exasol/nb_connector/slc/script_language_container.py +++ b/exasol/nb_connector/slc/script_language_container.py @@ -7,14 +7,15 @@ ) from exasol.slc import api as exaslct_api +from exasol.slc.models.compression_strategy import CompressionStrategy from exasol_integration_test_docker_environment.lib.docker import ( ContextDockerClient, ) from exasol.nb_connector.ai_lab_config import AILabConfig as CKey -from exasol.nb_connector.language_container_activation import ACTIVATION_KEY_PREFIX from exasol.nb_connector.secret_store import Secrets from exasol.nb_connector.slc import constants +from exasol.nb_connector.slc.slc_compression_strategy import SlcCompressionStrategy from exasol.nb_connector.slc.slc_flavor import ( SlcError, SlcFlavor, @@ -25,6 +26,28 @@ ) PipPackageDefinition = namedtuple("PipPackageDefinition", ["pkg", "version"]) +CondaPackageDefinition = namedtuple("CondaPackageDefinition", ["pkg", "version"]) + +NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]*$", flags=re.IGNORECASE) + + +def _verify_name(slc_name: str) -> None: + if not NAME_PATTERN.match(slc_name): + raise SlcError( + f'SLC name "{slc_name}" doesn\'t match' + f' regular expression "{NAME_PATTERN}".' + ) + + +def _append_packages( + file_path: Path, packages: list[PipPackageDefinition] | list[CondaPackageDefinition] +): + """ + Appends packages to the custom packages file. + """ + with open(file_path, "a") as f: + for p in packages: + print(f"{p.pkg}|{p.version}", file=f) class ScriptLanguageContainer: @@ -52,7 +75,9 @@ def __init__( ): self.secrets = secrets self.name = name + _verify_name(name) self.flavor = SlcFlavor(name).verify(secrets) + self.compression_strategy = SlcCompressionStrategy(name).verify(secrets) self.workspace = Workspace.for_slc(name) if not self.flavor_path.is_dir(): raise SlcError( @@ -65,14 +90,23 @@ def create( secrets: Secrets, name: str, flavor: str, + compression_strategy: CompressionStrategy = CompressionStrategy.GZIP, ) -> ScriptLanguageContainer: + _verify_name(name) slc_flavor = SlcFlavor(name) if slc_flavor.exists(secrets): raise SlcError( "Secure Configuration Storage already contains a" f" flavor for SLC name {name}." ) + slc_compression_strategy = SlcCompressionStrategy(slc_name=name) + if slc_compression_strategy.exists(secrets): + raise SlcError( + "Secure Configuration Storage already contains a" + f" compression strategy for SLC name {name}." + ) slc_flavor.save(secrets, flavor) + slc_compression_strategy.save(secrets, compression_strategy) workspace = Workspace.for_slc(name) workspace.clone_slc_repo() return cls(secrets=secrets, name=name) @@ -96,17 +130,26 @@ def _flavor_path_rel(self) -> str: def flavor_path(self) -> Path: return self.checkout_dir / constants.FLAVORS_PATH_IN_SLC_REPO / self.flavor + @property + def custom_packages_dir(self): + """ + Returns the path to the custom packages directory of the flavor + """ + return self.flavor_path / "flavor_customization" / "packages" + @property def custom_pip_file(self) -> Path: """ - Returns the path to the custom pip file of the flavor + Returns the path to the custom pip packages file of the flavor """ - return ( - self.flavor_path - / "flavor_customization" - / "packages" - / "python3_pip_packages" - ) + return self.custom_packages_dir / "python3_pip_packages" + + @property + def custom_conda_file(self) -> Path: + """ + Returns the path to the custom conda packages file of the flavor + """ + return self.custom_packages_dir / "conda_packages" def export(self): """ @@ -118,6 +161,7 @@ def export(self): export_path=str(self.workspace.export_path), output_directory=str(self.workspace.output_path), release_name=self.language_alias, + compression_strategy=self.compression_strategy, ) def deploy(self): @@ -144,6 +188,7 @@ def deploy(self): path_in_bucket=constants.PATH_IN_BUCKET, release_name=self.language_alias, output_directory=str(self.workspace.output_path), + compression_strategy=self.compression_strategy, ) deploy_result = result[self._flavor_path_rel]["release"] builder = deploy_result.language_definition_builder @@ -170,16 +215,23 @@ def activation_key(self) -> str: "Secure Configuration Storage does not contains an activation key." ) from ex - def append_custom_packages(self, pip_packages: list[PipPackageDefinition]): + def append_custom_pip_packages(self, pip_packages: list[PipPackageDefinition]): + """ + Appends packages to the custom pip packages file. + Note: This method is not idempotent: Multiple calls with the same + package definitions will result in duplicated entries. """ - Appends packages to the custom pip file. + _append_packages(self.custom_pip_file, pip_packages) + def append_custom_conda_packages( + self, conda_packages: list[CondaPackageDefinition] + ): + """ + Appends packages to the custom conda packages file. Note: This method is not idempotent: Multiple calls with the same - package definitions will result in duplicate entries. + package definitions will result in duplicated entries. """ - with open(self.custom_pip_file, "a") as f: - for p in pip_packages: - print(f"{p.pkg}|{p.version}", file=f) + _append_packages(self.custom_conda_file, conda_packages) @property def docker_image_tags(self) -> list[str]: diff --git a/exasol/nb_connector/slc/slc_compression_strategy.py b/exasol/nb_connector/slc/slc_compression_strategy.py new file mode 100644 index 00000000..26a6f7c1 --- /dev/null +++ b/exasol/nb_connector/slc/slc_compression_strategy.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from exasol.slc.models.compression_strategy import CompressionStrategy + +from exasol.nb_connector.secret_store import Secrets +from exasol.nb_connector.slc.slc_error import SlcError + + +class SlcCompressionStrategy: + def __init__(self, slc_name): + self.slc_name = slc_name + + @property + def key(self): + return f"SLC_COMPRESSION_STRATEGY_{self.slc_name.upper()}" + + def save(self, secrets: Secrets, compression_strategy: CompressionStrategy) -> None: + secrets.save(self.key, compression_strategy.value) + + def exists(self, secrets: Secrets) -> bool: + return True if secrets.get(self.key) else False + + def verify(self, secrets: Secrets) -> CompressionStrategy: + try: + value = secrets[self.key] + return CompressionStrategy(value) + except AttributeError as ex: + raise SlcError( + "Secure Configuration Storage does not contain a" + f" compression strategy for SLC {self.slc_name}." + ) from ex diff --git a/exasol/nb_connector/slc/slc_error.py b/exasol/nb_connector/slc/slc_error.py new file mode 100644 index 00000000..c3a044f7 --- /dev/null +++ b/exasol/nb_connector/slc/slc_error.py @@ -0,0 +1,11 @@ +class SlcError(Exception): + """ + Signals errors related to ScriptLanguageContainer: + + * The name of the SLC is not unique + + * the Secure Configuration Storage (SCS / secrets / conf) does not contain + a required option + + * The SLC Git repository has not been checked out (cloned) + """ diff --git a/exasol/nb_connector/slc/slc_flavor.py b/exasol/nb_connector/slc/slc_flavor.py index a33526b6..fc558506 100644 --- a/exasol/nb_connector/slc/slc_flavor.py +++ b/exasol/nb_connector/slc/slc_flavor.py @@ -1,33 +1,11 @@ from __future__ import annotations -import re -from pathlib import Path - from exasol.nb_connector.secret_store import Secrets - -NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]*$", flags=re.IGNORECASE) - - -class SlcError(Exception): - """ - Signals errors related to ScriptLanguageContainer: - - * The name of the SLC is not unique - - * the Secure Configuration Storage (SCS / secrets / conf) does not contain - a required option - - * The SLC Git repository has not been checked out (cloned) - """ +from exasol.nb_connector.slc.slc_error import SlcError class SlcFlavor: def __init__(self, slc_name: str): - if not NAME_PATTERN.match(slc_name): - raise SlcError( - f'SLC name "{slc_name}" doesn\'t match' - f' regular expression "{NAME_PATTERN}".' - ) self.slc_name = slc_name @property diff --git a/test/conftest.py b/test/conftest.py index 0efa29f5..e9417bab 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,15 +1,52 @@ -from pathlib import Path +from collections.abc import Iterator +from test.package_manager import PackageManager +from test.utils.integration_test_utils import sample_db_file import pytest +from exasol.slc.models.compression_strategy import CompressionStrategy from exasol.nb_connector.secret_store import Secrets @pytest.fixture -def sample_file(tmp_path: Path) -> Path: - return tmp_path / "sample_database.db" +def secrets() -> Iterator[Secrets]: + with sample_db_file() as secret_db: + yield Secrets(secret_db, master_password="abc") -@pytest.fixture -def secrets(sample_file) -> Secrets: # pylint: disable=W0621 - return Secrets(sample_file, master_password="abc") +@pytest.fixture(scope="module") +def secrets_module() -> Iterator[Secrets]: + with sample_db_file() as secret_db: + yield Secrets(secret_db, master_password="abc") + + +def pytest_addoption(parser): + parser.addoption( + "--package-manager", + action="store", + type=PackageManager, + choices=list(PackageManager), + default="pip", + help="Package Manager to use", + ) + + parser.addoption( + "--compression-strategy", + action="store", + type=CompressionStrategy, + choices=list(CompressionStrategy), + default="gzip", + help="Compression Strategy to use", + ) + + +@pytest.fixture(scope="session") +def package_manager(request) -> PackageManager: + val = request.config.getoption("--package-manager") + return PackageManager(val) + + +@pytest.fixture(scope="session") +def compression_strategy(request) -> CompressionStrategy: + val = request.config.getoption("--compression-strategy") + return CompressionStrategy(val) diff --git a/test/integration/gpu/itest_slct_manager_with_gpu.py b/test/integration/gpu/itest_slct_manager_with_gpu.py new file mode 100644 index 00000000..e46bc47d --- /dev/null +++ b/test/integration/gpu/itest_slct_manager_with_gpu.py @@ -0,0 +1,107 @@ +import textwrap +from collections.abc import Iterator +from test.utils.integration_test_utils import ( + sample_db_file, + setup_itde_module, +) + +import pytest +from exasol.slc.models.compression_strategy import CompressionStrategy + +from exasol.nb_connector.ai_lab_config import ( + Accelerator, + AILabConfig, +) +from exasol.nb_connector.language_container_activation import ( + open_pyexasol_connection_with_lang_definitions, +) +from exasol.nb_connector.secret_store import Secrets +from exasol.nb_connector.slc import ScriptLanguageContainer +from exasol.nb_connector.slc.script_language_container import CondaPackageDefinition + +DEFAULT_GPU_FLAVOR = "template-Exasol-8-python-3.10-cuda-conda" +""" +The flavor may depend on the release of the SLCR used via SLC_RELEASE_TAG in constants.py. +See the developer guide (./doc/developer-guide.md) for more details. +""" + + +@pytest.fixture(scope="module") +def secrets_module() -> Iterator[Secrets]: + with sample_db_file() as secret_db: + secrets = Secrets(secret_db, master_password="abc") + secrets.save(AILabConfig.accelerator, Accelerator.nvidia.value) + yield secrets + + +@pytest.fixture(scope="module") +def sample_slc(secrets_module: Secrets) -> ScriptLanguageContainer: + return ScriptLanguageContainer.create( + secrets_module, + name="sample_gpu", + flavor=DEFAULT_GPU_FLAVOR, + compression_strategy=CompressionStrategy.NONE, + ) + + +@pytest.fixture +def custom_packages() -> list[tuple[str, str]]: + return [("numba[cuda]", "0.61.2")] + + +@pytest.mark.dependency(name="append_custom_packages") +def test_append_custom_packages( + sample_slc: ScriptLanguageContainer, + custom_packages: list[tuple[str, str]], +): + sample_slc.append_custom_conda_packages( + [CondaPackageDefinition(pkg, version) for pkg, version in custom_packages] + ) + + +@pytest.mark.dependency( + name="upload_slc_with_new_packages", depends=["append_custom_packages"] +) +def test_upload_slc_with_new_packages( + secrets_module: Secrets, + setup_itde_module, + sample_slc: ScriptLanguageContainer, +): + sample_slc.deploy() + assert ( + sample_slc.activation_key + == f"{sample_slc.language_alias}=localzmq+protobuf:///bfsdefault/default/container/{DEFAULT_GPU_FLAVOR}-release-{sample_slc.language_alias}?lang=python#buckets/bfsdefault/default/container/{DEFAULT_GPU_FLAVOR}-release-{sample_slc.language_alias}/exaudf/exaudfclient" + ) + + +@pytest.mark.dependency( + name="udf_with_new_packages", depends=["upload_slc_with_new_packages"] +) +def test_numba( + secrets_module: Secrets, + sample_slc: ScriptLanguageContainer, +): + udf = textwrap.dedent( + f""" + CREATE OR REPLACE {sample_slc.language_alias} SCALAR SCRIPT + test_gpu_available() + RETURNS VARCHAR(1000) AS + %perInstanceRequiredAcceleratorDevices GpuNvidia; + from numba import cuda + def run(ctx): + if cuda.is_available(): + return "GPU Found" + else: + return "GPU Not Found" + / + """ + ) + con = open_pyexasol_connection_with_lang_definitions(secrets_module) + try: + con.execute("CREATE SCHEMA TEST") + con.execute(udf) + res = con.execute("select test_gpu_available()") + rows = res.fetchall() + assert rows == [("GPU Found",)] + finally: + con.close() diff --git a/test/integration/gpu/test_itde_manager_with_gpu.py b/test/integration/gpu/test_itde_manager_with_gpu.py index 802a7682..c0e2f4d2 100644 --- a/test/integration/gpu/test_itde_manager_with_gpu.py +++ b/test/integration/gpu/test_itde_manager_with_gpu.py @@ -24,7 +24,7 @@ def test_itde_with_gpu(secrets): query_accelerator_parameters = cleandoc( f""" SELECT PARAM_VALUE, PARAM_NAME FROM EXA_METADATA - WHERE PARAM_NAME LIKE '%accelerator%' + WHERE PARAM_NAME IN ('acceleratorDeviceDetected', 'acceleratorDeviceGpuNvidiaDetected') ORDER BY PARAM_NAME; """ ) diff --git a/test/integration/ordinary/itest_slc.py b/test/integration/ordinary/itest_slc.py index 008712e5..4ed44bd9 100644 --- a/test/integration/ordinary/itest_slc.py +++ b/test/integration/ordinary/itest_slc.py @@ -1,49 +1,28 @@ import textwrap -from pathlib import Path -from tempfile import TemporaryDirectory -from test.integration.ordinary.test_itde_manager import remove_itde +from test.package_manager import PackageManager +from test.utils.integration_test_utils import setup_itde_module import pytest from docker.models.images import Image as DockerImage +from exasol.slc.models.compression_strategy import CompressionStrategy from exasol_integration_test_docker_environment.lib.docker import ContextDockerClient -from exasol.nb_connector.itde_manager import bring_itde_up from exasol.nb_connector.language_container_activation import ( open_pyexasol_connection_with_lang_definitions, ) from exasol.nb_connector.secret_store import Secrets from exasol.nb_connector.slc.script_language_container import ( + CondaPackageDefinition, PipPackageDefinition, ScriptLanguageContainer, constants, ) +DEFAULT_FLAVORS = { + PackageManager.PIP: "template-Exasol-all-python-3.10", + PackageManager.CONDA: "template-Exasol-all-python-3.10-conda", +} -@pytest.fixture(scope="module") -def working_path() -> Path: - with TemporaryDirectory() as d: - yield Path(d) - - -@pytest.fixture(scope="module") -def secrets_file(working_path: Path) -> Path: - return working_path / "sample_database.db" - - -@pytest.fixture(scope="module") -def slc_secrets(secrets_file, working_path) -> Secrets: - secrets = Secrets(secrets_file, master_password="abc") - return secrets - - -@pytest.fixture(scope="module") -def itde(slc_secrets: Secrets): - bring_itde_up(slc_secrets) - yield - remove_itde() - - -DEFAULT_FLAVOR = "template-Exasol-all-python-3.10" OTHER_FLAVOR = "template-Exasol-all-r-4" """ The flavors may depend on the release of the SLCR used via SLC_RELEASE_TAG in constants.py. @@ -51,27 +30,46 @@ def itde(slc_secrets: Secrets): """ +@pytest.fixture(scope="module") +def default_flavor(package_manager: PackageManager) -> str: + return DEFAULT_FLAVORS[package_manager] + + def create_slc( secrets: Secrets, name: str, - flavor: str = DEFAULT_FLAVOR, + flavor: str, + compression_strategy: CompressionStrategy, ) -> ScriptLanguageContainer: - return ScriptLanguageContainer.create(secrets, name=name, flavor=flavor) + return ScriptLanguageContainer.create( + secrets, name=name, flavor=flavor, compression_strategy=compression_strategy + ) @pytest.fixture(scope="module") -def sample_slc(slc_secrets: Secrets, working_path: Path) -> ScriptLanguageContainer: - return create_slc(slc_secrets, "sample") +def sample_slc( + secrets_module: Secrets, + default_flavor: str, + compression_strategy: CompressionStrategy, +) -> ScriptLanguageContainer: + return create_slc(secrets_module, "sample", default_flavor, compression_strategy) @pytest.fixture(scope="module") -def other_slc(slc_secrets: Secrets, working_path: Path) -> ScriptLanguageContainer: +def other_slc( + secrets_module: Secrets, compression_strategy: CompressionStrategy +) -> ScriptLanguageContainer: """ Creates another SLC with a different flavor for verifying operations to be limited to the current SLC only, e.g. removing docker images or working directories. """ - slc = create_slc(slc_secrets, "other", flavor=OTHER_FLAVOR) + slc = create_slc( + secrets_module, + "other", + flavor=OTHER_FLAVOR, + compression_strategy=compression_strategy, + ) slc.export() slc.deploy() return slc @@ -83,16 +81,21 @@ def custom_packages() -> list[tuple[str, str, str]]: @pytest.mark.dependency(name="export_slc") -def test_export_slc(sample_slc: ScriptLanguageContainer): +def test_export_slc( + sample_slc: ScriptLanguageContainer, compression_strategy: CompressionStrategy +): sample_slc.export() export_path = sample_slc.workspace.export_path + expected_suffix = ( + "tar" if compression_strategy == CompressionStrategy.NONE else "tar.gz" + ) assert export_path.exists() - tgz = [f for f in export_path.glob("*.tar.gz")] - assert len(tgz) == 1 - assert tgz[0].is_file() - tgz_sum = [f for f in export_path.glob("*.tar.gz.sha512sum")] - assert len(tgz_sum) == 1 - assert tgz_sum[0].is_file() + tar = [f for f in export_path.glob(f"*.{expected_suffix}")] + assert len(tar) == 1 + assert tar[0].is_file() + tar_sum = [f for f in export_path.glob(f"*.{expected_suffix}.sha512sum")] + assert len(tar_sum) == 1 + assert tar_sum[0].is_file() def slc_docker_tag_prefix(slc: ScriptLanguageContainer) -> str: @@ -118,28 +121,55 @@ def expected_activation_key(slc: ScriptLanguageContainer) -> str: @pytest.mark.dependency(name="deploy_slc") -def test_deploy(sample_slc: ScriptLanguageContainer, itde): +def test_deploy(sample_slc: ScriptLanguageContainer, setup_itde_module): sample_slc.deploy() assert sample_slc.activation_key == expected_activation_key(sample_slc) -@pytest.mark.dependency(name="append_custom_packages", depends=["deploy_slc"]) -def test_append_custom_packages( - sample_slc: ScriptLanguageContainer, custom_packages: list[tuple[str, str, str]] +@pytest.mark.dependency(name="append_custom_pip_packages", depends=["deploy_slc"]) +def test_append_custom_pip_packages( + sample_slc: ScriptLanguageContainer, + custom_packages: list[tuple[str, str, str]], + package_manager: PackageManager, ): - sample_slc.append_custom_packages( - [PipPackageDefinition(pkg, version) for pkg, version, _ in custom_packages] - ) - with open(sample_slc.custom_pip_file) as f: - pip_content = f.read() - for custom_package, version, _ in custom_packages: - assert f"{custom_package}|{version}" in pip_content + # Cannot skip the test if it's not a Pip package manager, otherwise dependent tests below won't run + if package_manager == PackageManager.PIP: + sample_slc.append_custom_pip_packages( + [PipPackageDefinition(pkg, version) for pkg, version, _ in custom_packages] + ) + with open(sample_slc.custom_pip_file) as f: + pip_content = f.read() + for custom_package, version, _ in custom_packages: + assert f"{custom_package}|{version}" in pip_content + + +@pytest.mark.dependency(name="append_custom_conda_packages", depends=["deploy_slc"]) +def test_append_custom_conda_packages( + sample_slc: ScriptLanguageContainer, + custom_packages: list[tuple[str, str, str]], + package_manager: PackageManager, +): + # Cannot skip the test if it's not a Conda package manager, otherwise dependent tests below won't run + if package_manager == PackageManager.CONDA: + sample_slc.append_custom_conda_packages( + [ + CondaPackageDefinition(pkg, version) + for pkg, version, _ in custom_packages + ] + ) + with open(sample_slc.custom_conda_file) as f: + conda_content = f.read() + for custom_package, version, _ in custom_packages: + assert f"{custom_package}|{version}" in conda_content @pytest.mark.dependency( - name="deploy_slc_with_custom_packages", depends=["append_custom_packages"] + name="deploy_slc_with_custom_packages", + depends=["append_custom_pip_packages", "append_custom_conda_packages"], ) -def test_deploy_slc_with_custom_packages(sample_slc: ScriptLanguageContainer): +def test_deploy_slc_with_custom_packages( + sample_slc: ScriptLanguageContainer, setup_itde_module +): sample_slc.deploy() assert sample_slc.activation_key == expected_activation_key(sample_slc) @@ -149,7 +179,7 @@ def test_deploy_slc_with_custom_packages(sample_slc: ScriptLanguageContainer): depends=["deploy_slc_with_custom_packages"], ) def test_udf_with_custom_packages( - slc_secrets: Secrets, + secrets_module: Secrets, sample_slc: ScriptLanguageContainer, custom_packages: list[tuple[str, str, str]], ): @@ -171,7 +201,7 @@ def run(ctx): language_alias=sample_slc.language_alias, import_statements=import_statements, ) - con = open_pyexasol_connection_with_lang_definitions(slc_secrets) + con = open_pyexasol_connection_with_lang_definitions(secrets_module) try: con.execute("CREATE SCHEMA TEST") con.execute(udf) diff --git a/test/package_manager.py b/test/package_manager.py new file mode 100644 index 00000000..c80475f5 --- /dev/null +++ b/test/package_manager.py @@ -0,0 +1,14 @@ +import enum + + +class PackageManager(enum.Enum): + """ + Supported Python package managers. + """ + + PIP = "pip" + CONDA = "conda" + + def __str__(self): + # this allows using `choices` in argparse + return self.value diff --git a/test/unit/slc/utest_slc.py b/test/unit/slc/utest_slc.py index 6be4d853..a45501ea 100644 --- a/test/unit/slc/utest_slc.py +++ b/test/unit/slc/utest_slc.py @@ -12,6 +12,7 @@ import git import pytest from _pytest.monkeypatch import MonkeyPatch +from exasol.slc.models.compression_strategy import CompressionStrategy from exasol.nb_connector.secret_store import Secrets from exasol.nb_connector.slc import ( @@ -109,6 +110,7 @@ def test_create( "packages", "python3_pip_packages", ) + assert testee.compression_strategy == CompressionStrategy.GZIP def test_repo_missing(sample_slc_name): diff --git a/test/unit/slc/util.py b/test/unit/slc/util.py index 4413e49c..68fbadc4 100644 --- a/test/unit/slc/util.py +++ b/test/unit/slc/util.py @@ -3,8 +3,10 @@ import contextlib import pytest +from exasol.slc.models.compression_strategy import CompressionStrategy from exasol.nb_connector.secret_store import Secrets +from exasol.nb_connector.slc.slc_compression_strategy import SlcCompressionStrategy from exasol.nb_connector.slc.slc_flavor import SlcFlavor @@ -34,10 +36,12 @@ def for_slc( cls, slc_name: str, flavor: str | None = "Vanilla", + compression_strategy: CompressionStrategy = CompressionStrategy.NONE, ) -> SecretsMock: instance = cls(slc_name) if flavor: SlcFlavor(slc_name).save(instance, flavor) + SlcCompressionStrategy(slc_name).save(instance, compression_strategy) return instance diff --git a/test/unit/test_secret_store.py b/test/unit/test_secret_store.py index b29c3c90..6080bcc3 100644 --- a/test/unit/test_secret_store.py +++ b/test/unit/test_secret_store.py @@ -1,4 +1,5 @@ import sqlite3 +from pathlib import Path import pytest @@ -9,6 +10,11 @@ ) +@pytest.fixture +def sample_file(tmp_path: Path) -> Path: + return tmp_path / "sample_database.db" + + def test_no_database_file(secrets): assert not secrets.db_file.exists() diff --git a/test/utils/integration_test_utils.py b/test/utils/integration_test_utils.py index 7d92c894..ca09d144 100644 --- a/test/utils/integration_test_utils.py +++ b/test/utils/integration_test_utils.py @@ -1,7 +1,11 @@ from __future__ import annotations +import contextlib import textwrap +from collections.abc import Iterator from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory import pytest from pyexasol import ExaConnection @@ -16,13 +20,13 @@ from exasol.nb_connector.secret_store import Secrets -@pytest.fixture -def setup_itde(secrets) -> None: - """ - Brings up the ITDE and takes it down when the tests are completed or failed. - Creates a schema and saves its name in the secret store. - """ +@contextlib.contextmanager +def sample_db_file() -> Iterator[Path]: + with TemporaryDirectory() as d: + yield Path(d) / "sample_database.db" + +def _setup_itde_impl(secrets: Secrets) -> Iterator[None]: bring_itde_up(secrets) schema = "INTEGRATION_TEST" @@ -36,6 +40,26 @@ def setup_itde(secrets) -> None: take_itde_down(secrets) +@pytest.fixture +def setup_itde(secrets) -> Iterator[None]: + """ + Brings up the ITDE and takes it down when the tests are completed or failed. + Creates a schema and saves its name in the secret store. + The scope is per test function. + """ + yield from _setup_itde_impl(secrets) + + +@pytest.fixture(scope="module") +def setup_itde_module(secrets_module) -> Iterator[None]: + """ + Brings up the ITDE and takes it down when the tests are completed or failed. + Creates a schema and saves its name in the secret store. + The scope is per test module. + """ + yield from _setup_itde_impl(secrets_module) + + def activate_languages(pyexasol_connection: ExaConnection, secrets: Secrets) -> None: """ Activates languages at the current session level.