#219: Notebook-Connector can add Conda-Packages to SLCs

.github/workflows/tests-gpu.yml

@@ -9,6 +9,7 @@ jobs:
     name: GPU Tests
     runs-on:
       labels: int-linux-x64-4core-gpu-t4-ubuntu24.04-1
+
     permissions:
       contents: read
 
@@ -43,4 +44,4 @@ jobs:
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
 
       - name: Tests
-        run: poetry run -- pytest -rA --setup-show test/integration/gpu
+        run: poetry run -- pytest -rA --setup-show test/integration/gpu/

.github/workflows/tests-ordinary-integration.yml

@@ -13,16 +13,29 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        file:
-          - test_itde_manager.py
-          - test_sagemaker_extension_wrapper.py
-          - itest_slc.py
-          - test_transformers_extension_wrapper.py
-          - itde_mgr_in_container/test_itde_connect.py
-          - itde_mgr_in_container/test_itde_external.py
-          - itde_mgr_in_container/test_itde_recreation_after_take_down.py
-          - itde_mgr_in_container/test_itde_recreation_without_take_down.py
-          - itde_mgr_in_container/test_itde_stop_and_restart.py
+        include:
+          - file: test_itde_manager.py
+            extra_param: ""
+          - file: test_sagemaker_extension_wrapper.py
+            extra_param: ""
+          - file: test_transformers_extension_wrapper.py
+            extra_param: ""
+          - file: itde_mgr_in_container/test_itde_connect.py
+            extra_param: ""
+          - file: itde_mgr_in_container/test_itde_external.py
+            extra_param: ""
+          - file: itde_mgr_in_container/test_itde_recreation_after_take_down.py
+            extra_param: ""
+          - file: itde_mgr_in_container/test_itde_recreation_without_take_down.py
+            extra_param: ""
+          - file: itde_mgr_in_container/test_itde_stop_and_restart.py
+            extra_param: ""
+          - file: itest_slc.py
+            extra_param: "--package-manager pip --compression-strategy gzip"
+          - file: itest_slc.py
+            extra_param: "--package-manager conda --compression-strategy gzip"
+          - file: itest_slc.py
+            extra_param: "--package-manager pip --compression-strategy none"
 
     steps:
 
@@ -53,4 +66,4 @@ jobs:
           sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
 
       - name: Pytest
-        run: poetry run -- pytest -rA --setup-show test/integration/ordinary/${{ matrix.file }}
+        run: poetry run -- pytest -rA --setup-show test/integration/ordinary/${{ matrix.file }} ${{matrix.extra_param}}

.gitignore

@@ -11,3 +11,4 @@ __pycache__/
 .build_output
 # drawio backup files
 /doc/user_guide/*.bkp
+.workspace

exasol/nb_connector/slc/__init__.py

@@ -1,5 +1,6 @@
 from exasol.nb_connector.slc.script_language_container import (
+    CondaPackageDefinition,
     PipPackageDefinition,
     ScriptLanguageContainer,
 )
-from exasol.nb_connector.slc.slc_flavor import SlcError
+from exasol.nb_connector.slc.slc_error import SlcError

exasol/nb_connector/slc/script_language_container.py

@@ -7,14 +7,15 @@
 )
 
 from exasol.slc import api as exaslct_api
+from exasol.slc.models.compression_strategy import CompressionStrategy
 from exasol_integration_test_docker_environment.lib.docker import (
     ContextDockerClient,
 )
 
 from exasol.nb_connector.ai_lab_config import AILabConfig as CKey
-from exasol.nb_connector.language_container_activation import ACTIVATION_KEY_PREFIX
 from exasol.nb_connector.secret_store import Secrets
 from exasol.nb_connector.slc import constants
+from exasol.nb_connector.slc.slc_compression_strategy import SlcCompressionStrategy
 from exasol.nb_connector.slc.slc_flavor import (
     SlcError,
     SlcFlavor,
@@ -25,6 +26,28 @@
 )
 
 PipPackageDefinition = namedtuple("PipPackageDefinition", ["pkg", "version"])
+CondaPackageDefinition = namedtuple("CondaPackageDefinition", ["pkg", "version"])
+
+NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]*$", flags=re.IGNORECASE)
+
+
+def _verify_name(slc_name: str) -> None:
+    if not NAME_PATTERN.match(slc_name):
+        raise SlcError(
+            f'SLC name "{slc_name}" doesn\'t match'
+            f' regular expression "{NAME_PATTERN}".'
+        )
+
+
+def _append_packages(
+    file_path: Path, packages: list[PipPackageDefinition] | list[CondaPackageDefinition]
+):
+    """
+    Appends packages to the custom packages file.
+    """
+    with open(file_path, "a") as f:
+        for p in packages:
+            print(f"{p.pkg}|{p.version}", file=f)
 
 
 class ScriptLanguageContainer:
@@ -52,7 +75,9 @@ def __init__(
     ):
         self.secrets = secrets
         self.name = name
+        _verify_name(name)
         self.flavor = SlcFlavor(name).verify(secrets)
+        self.compression_strategy = SlcCompressionStrategy(name).verify(secrets)
         self.workspace = Workspace.for_slc(name)
         if not self.flavor_path.is_dir():
             raise SlcError(
@@ -65,14 +90,23 @@ def create(
         secrets: Secrets,
         name: str,
         flavor: str,
+        compression_strategy: CompressionStrategy = CompressionStrategy.GZIP,
     ) -> ScriptLanguageContainer:
+        _verify_name(name)
         slc_flavor = SlcFlavor(name)
         if slc_flavor.exists(secrets):
             raise SlcError(
                 "Secure Configuration Storage already contains a"
                 f" flavor for SLC name {name}."
             )
+        slc_compression_strategy = SlcCompressionStrategy(slc_name=name)
+        if slc_compression_strategy.exists(secrets):
+            raise SlcError(
+                "Secure Configuration Storage already contains a"
+                f" compression strategy for SLC name {name}."
+            )
         slc_flavor.save(secrets, flavor)
+        slc_compression_strategy.save(secrets, compression_strategy)
         workspace = Workspace.for_slc(name)
         workspace.clone_slc_repo()
         return cls(secrets=secrets, name=name)
@@ -96,17 +130,26 @@ def _flavor_path_rel(self) -> str:
     def flavor_path(self) -> Path:
         return self.checkout_dir / constants.FLAVORS_PATH_IN_SLC_REPO / self.flavor
 
+    @property
+    def custom_packages_dir(self):
+        """
+        Returns the path to the custom packages directory of the flavor
+        """
+        return self.flavor_path / "flavor_customization" / "packages"
+
     @property
     def custom_pip_file(self) -> Path:
         """
-        Returns the path to the custom pip file of the flavor
+        Returns the path to the custom pip packages file of the flavor
         """
-        return (
-            self.flavor_path
-            / "flavor_customization"
-            / "packages"
-            / "python3_pip_packages"
-        )
+        return self.custom_packages_dir / "python3_pip_packages"
+
+    @property
+    def custom_conda_file(self) -> Path:
+        """
+        Returns the path to the custom conda packages file of the flavor
+        """
+        return self.custom_packages_dir / "conda_packages"
 
     def export(self):
         """
@@ -118,6 +161,7 @@ def export(self):
                 export_path=str(self.workspace.export_path),
                 output_directory=str(self.workspace.output_path),
                 release_name=self.language_alias,
+                compression_strategy=self.compression_strategy,
             )
 
     def deploy(self):
@@ -144,6 +188,7 @@ def deploy(self):
                 path_in_bucket=constants.PATH_IN_BUCKET,
                 release_name=self.language_alias,
                 output_directory=str(self.workspace.output_path),
+                compression_strategy=self.compression_strategy,
             )
             deploy_result = result[self._flavor_path_rel]["release"]
             builder = deploy_result.language_definition_builder
@@ -170,16 +215,23 @@ def activation_key(self) -> str:
                 "Secure Configuration Storage does not contains an activation key."
             ) from ex
 
-    def append_custom_packages(self, pip_packages: list[PipPackageDefinition]):
+    def append_custom_pip_packages(self, pip_packages: list[PipPackageDefinition]):
+        """
+        Appends packages to the custom pip packages file.
+        Note: This method is not idempotent: Multiple calls with the same
+        package definitions will result in duplicated entries.
         """
-        Appends packages to the custom pip file.
+        _append_packages(self.custom_pip_file, pip_packages)
 
+    def append_custom_conda_packages(
+        self, conda_packages: list[CondaPackageDefinition]
+    ):
+        """
+        Appends packages to the custom conda packages file.
         Note: This method is not idempotent: Multiple calls with the same
-        package definitions will result in duplicate entries.
+        package definitions will result in duplicated entries.
         """
-        with open(self.custom_pip_file, "a") as f:
-            for p in pip_packages:
-                print(f"{p.pkg}|{p.version}", file=f)
+        _append_packages(self.custom_conda_file, conda_packages)
 
     @property
     def docker_image_tags(self) -> list[str]:

exasol/nb_connector/slc/slc_compression_strategy.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from exasol.slc.models.compression_strategy import CompressionStrategy
+
+from exasol.nb_connector.secret_store import Secrets
+from exasol.nb_connector.slc.slc_error import SlcError
+
+
+class SlcCompressionStrategy:
+    def __init__(self, slc_name):
+        self.slc_name = slc_name
+
+    @property
+    def key(self):
+        return f"SLC_COMPRESSION_STRATEGY_{self.slc_name.upper()}"
+
+    def save(self, secrets: Secrets, compression_strategy: CompressionStrategy) -> None:
+        secrets.save(self.key, compression_strategy.value)
+
+    def exists(self, secrets: Secrets) -> bool:
+        return True if secrets.get(self.key) else False
+
+    def verify(self, secrets: Secrets) -> CompressionStrategy:
+        try:
+            value = secrets[self.key]
+            return CompressionStrategy(value)
+        except AttributeError as ex:
+            raise SlcError(
+                "Secure Configuration Storage does not contain a"
+                f" compression strategy for SLC {self.slc_name}."
+            ) from ex

exasol/nb_connector/slc/slc_error.py

@@ -0,0 +1,11 @@
+class SlcError(Exception):
+    """
+    Signals errors related to ScriptLanguageContainer:
+
+    * The name of the SLC is not unique
+
+    * the Secure Configuration Storage (SCS / secrets / conf) does not contain
+      a required option
+
+    * The SLC Git repository has not been checked out (cloned)
+    """

exasol/nb_connector/slc/slc_flavor.py

@@ -1,33 +1,11 @@
 from __future__ import annotations
 
-import re
-from pathlib import Path
-
 from exasol.nb_connector.secret_store import Secrets
-
-NAME_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]*$", flags=re.IGNORECASE)
-
-
-class SlcError(Exception):
-    """
-    Signals errors related to ScriptLanguageContainer:
-
-    * The name of the SLC is not unique
-
-    * the Secure Configuration Storage (SCS / secrets / conf) does not contain
-      a required option
-
-    * The SLC Git repository has not been checked out (cloned)
-    """
+from exasol.nb_connector.slc.slc_error import SlcError
 
 
 class SlcFlavor:
     def __init__(self, slc_name: str):
-        if not NAME_PATTERN.match(slc_name):
-            raise SlcError(
-                f'SLC name "{slc_name}" doesn\'t match'
-                f' regular expression "{NAME_PATTERN}".'
-            )
         self.slc_name = slc_name
 
     @property

test/conftest.py

@@ -1,15 +1,52 @@
-from pathlib import Path
+from collections.abc import Iterator
+from test.package_manager import PackageManager
+from test.utils.integration_test_utils import sample_db_file
 
 import pytest
+from exasol.slc.models.compression_strategy import CompressionStrategy
 
 from exasol.nb_connector.secret_store import Secrets
 
 
 @pytest.fixture
-def sample_file(tmp_path: Path) -> Path:
-    return tmp_path / "sample_database.db"
+def secrets() -> Iterator[Secrets]:
+    with sample_db_file() as secret_db:
+        yield Secrets(secret_db, master_password="abc")
 
 
-@pytest.fixture
-def secrets(sample_file) -> Secrets:  # pylint: disable=W0621
-    return Secrets(sample_file, master_password="abc")
+@pytest.fixture(scope="module")
+def secrets_module() -> Iterator[Secrets]:
+    with sample_db_file() as secret_db:
+        yield Secrets(secret_db, master_password="abc")
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--package-manager",
+        action="store",
+        type=PackageManager,
+        choices=list(PackageManager),
+        default="pip",
+        help="Package Manager to use",
+    )
+
+    parser.addoption(
+        "--compression-strategy",
+        action="store",
+        type=CompressionStrategy,
+        choices=list(CompressionStrategy),
+        default="gzip",
+        help="Compression Strategy to use",
+    )
+
+
+@pytest.fixture(scope="session")
+def package_manager(request) -> PackageManager:
+    val = request.config.getoption("--package-manager")
+    return PackageManager(val)
+
+
+@pytest.fixture(scope="session")
+def compression_strategy(request) -> CompressionStrategy:
+    val = request.config.getoption("--compression-strategy")
+    return CompressionStrategy(val)