[Testing] Test SSH file transfer with images.

.github/workflows/code_test_and_deploy.yml

@@ -32,7 +32,7 @@ jobs:
         # macos-14 is M1, macos-13 is intel. Run on earliest and
         # latest python versions. All python versions are tested in
         # the weekly cron job.
-        os: [windows-latest, ubuntu-latest, macos-14, macos-13]
+        os: [ ubuntu-latest, windows-latest, macos-14, macos-13]
         # Test all Python versions for cron job, and only first/last for other triggers
         python-version: ${{ fromJson(github.event_name == 'schedule' && '["3.9", "3.10", "3.11", "3.12"]' || '["3.9", "3.12"]') }}
 
@@ -57,8 +57,17 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install .[dev]
-      - name: Test
-        run: pytest
+        # run SSH tests only on Linux because Windows and macOS
+        # are already run within a virtual container and so cannot
+        # run Linux containers because nested containerisation is disabled.
+      - name: Test SSH (Linux only)
+        if: runner.os == 'Linux'
+        run: |
+          sudo service mysql stop  # free up port 3306 for ssh tests
+          pytest tests/tests_transfers/ssh
+      - name: All Other Tests
+        run: |
+          pytest --ignore tests/tests_transfers/ssh
 
   build_sdist_wheels:
     name: Build source distribution

datashuttle/configs/canonical_configs.py

@@ -9,6 +9,7 @@
 
 from __future__ import annotations
 
+import os
 from typing import (
     TYPE_CHECKING,
     Dict,
@@ -52,6 +53,14 @@ def keys_str_on_file_but_path_in_class() -> list[str]:
     ]
 
 
+def get_default_ssh_port() -> int:
+    """Get the default port used for SSH connections."""
+    if "DS_SSH_PORT" in os.environ:
+        return int(os.environ["DS_SSH_PORT"])
+    else:
+        return 22
+
+
 # -----------------------------------------------------------------------------
 # Check Configs
 # -----------------------------------------------------------------------------

datashuttle/utils/data_transfer.py

@@ -164,7 +164,6 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]:
                     self.update_list_with_non_ses_sub_level_folders(
                         extra_folder_names, extra_filenames, sub
                     )
-
                     continue
 
                 # Datatype (sub and ses level) --------------------------------

datashuttle/utils/rclone.py

@@ -6,6 +6,7 @@
 from subprocess import CompletedProcess
 from typing import Dict, List, Literal
 
+from datashuttle.configs import canonical_configs
 from datashuttle.configs.config_class import Configs
 from datashuttle.utils import utils
 from datashuttle.utils.custom_types import TopLevelFolder
@@ -161,7 +162,7 @@ def setup_rclone_config_for_ssh(
         f"sftp "
         f"host {cfg['central_host_id']} "
         f"user {cfg['central_host_username']} "
-        f"port 22 "
+        f"port {canonical_configs.get_default_ssh_port()} "
         f"key_file {ssh_key_path.as_posix()}",
         pipe_std=True,
     )

datashuttle/utils/ssh.py

@@ -14,6 +14,7 @@
 
 import paramiko
 
+from datashuttle.configs import canonical_configs
 from datashuttle.utils import utils
 
 # -----------------------------------------------------------------------------
@@ -58,6 +59,7 @@ def connect_client_core(
             else None
         ),
         look_for_keys=True,
+        port=canonical_configs.get_default_ssh_port(),
     )
 
 
@@ -122,7 +124,9 @@ def get_remote_server_key(central_host_id: str):
 
     """
     transport: paramiko.Transport
-    with paramiko.Transport(central_host_id) as transport:
+    with paramiko.Transport(
+        (central_host_id, canonical_configs.get_default_ssh_port())
+    ) as transport:
         transport.connect()
         key = transport.get_remote_server_key()
     return key
@@ -148,7 +152,15 @@ def save_hostkey_locally(key, central_host_id, hostkeys_path) -> None:
 
     """
     client = paramiko.SSHClient()
-    client.get_host_keys().add(central_host_id, key.get_name(), key)
+
+    port = canonical_configs.get_default_ssh_port()
+    host_key = f"[{central_host_id}]:{port}" if port != 22 else central_host_id
+
+    client.get_host_keys().add(
+        host_key,
+        key.get_name(),
+        key,
+    )
     client.get_host_keys().save(hostkeys_path.as_posix())
 
 
@@ -242,15 +254,16 @@ def connect_client_with_logging(
                 f"Connection to {cfg['central_host_id']} made successfully."
             )
 
-    except Exception:
+    except Exception as e:
         utils.log_and_raise_error(
             f"Could not connect to server. Ensure that \n"
             f"1) You have run setup_ssh_connection() \n"
             f"2) You are on VPN network if required. \n"
             f"3) The central_host_id: {cfg['central_host_id']} is"
             f" correct.\n"
             f"4) The central username:"
-            f" {cfg['central_host_username']}, and password are correct.",
+            f" {cfg['central_host_username']}, and password are correct."
+            f"Original error: {e}",
             ConnectionError,
         )
 

pyproject.toml

@@ -126,7 +126,8 @@ select = [
     "D",        # pydocstyle
 ]
 per-file-ignores = { "tests/*" = [
-    "D"  # ignore docstring formatting in tests for now
+    "D",  # ignore docstring formatting in tests for now
+    "TID252"
 ], "examples/*" = [
     "D400", # first line should end with a period.
     "D415", # first line should end with a period, question mark...

tests/tests_integration/base.py → tests/base.py

@@ -1,7 +1,8 @@
 import warnings
 
 import pytest
-import test_utils
+
+from . import test_utils
 
 TEST_PROJECT_NAME = "test_project"
 

tests/test_utils.py

@@ -10,7 +10,6 @@
 from pathlib import Path
 
 import yaml
-from file_conflicts_pathtable import get_pathtable
 
 from datashuttle import DataShuttle
 from datashuttle.configs import canonical_configs, canonical_folders
@@ -146,18 +145,6 @@ def make_test_path(base_path, local_or_central, test_project_name):
     return Path(base_path) / local_or_central / test_project_name
 
 
-def create_all_pathtable_files(pathtable):
-    for i in range(pathtable.shape[0]):
-        filepath = pathtable["base_folder"][i] / pathtable["path"][i]
-        filepath.parents[0].mkdir(parents=True, exist_ok=True)
-        write_file(filepath, contents="test_entry")
-
-
-def quick_create_project(base_path):
-    pathtable = get_pathtable(base_path)
-    create_all_pathtable_files(pathtable)
-
-
 # -----------------------------------------------------------------------------
 # Test Configs
 # -----------------------------------------------------------------------------

tests/tests_integration/test_configs.py

@@ -1,13 +1,14 @@
 import os
 
 import pytest
-import test_utils
-from base import BaseTest
 
 from datashuttle import DataShuttle
 from datashuttle.utils import getters
 from datashuttle.utils.custom_exceptions import ConfigError
 
+from .. import test_utils
+from ..base import BaseTest
+
 
 class TestConfigs(BaseTest):
     # Test Errors

tests/tests_integration/test_create_folders.py

@@ -5,12 +5,13 @@
 from os.path import join
 
 import pytest
-import test_utils
-from base import BaseTest
 
 from datashuttle.configs import canonical_configs, canonical_folders
 from datashuttle.configs.canonical_tags import tags
 
+from .. import test_utils
+from ..base import BaseTest
+
 
 class TestCreateFolders(BaseTest):
     @pytest.mark.parametrize("project", ["local", "full"], indirect=True)

tests/tests_integration/test_datatypes.py

@@ -1,11 +1,12 @@
 import os
 
 import pytest
-import test_utils
-from base import BaseTest
 
 from datashuttle.configs import canonical_configs
 
+from .. import test_utils
+from ..base import BaseTest
+
 
 class TestDatatypes(BaseTest):
     """Tests for creating folders and transfer (very similar to other tests)

tests/tests_integration/test_formatting.py

@@ -1,9 +1,10 @@
 import pytest
-from base import BaseTest
 
 from datashuttle.utils import formatting
 from datashuttle.utils.custom_exceptions import NeuroBlueprintError
 
+from ..base import BaseTest
+
 
 class TestFormatting(BaseTest):
     @pytest.mark.parametrize("prefix", ["sub", "ses"])

tests/tests_integration/test_local_only_mode.py

@@ -1,13 +1,14 @@
 import shutil
 
 import pytest
-import test_utils
-from base import BaseTest
 
 from datashuttle.utils.custom_exceptions import (
     ConfigError,
 )
 
+from .. import test_utils
+from ..base import BaseTest
+
 TEST_PROJECT_NAME = "test_project"
 
 

tests/tests_integration/test_logging.py

@@ -5,7 +5,6 @@
 from pathlib import Path
 
 import pytest
-import test_utils
 
 from datashuttle.configs import canonical_configs
 from datashuttle.configs.canonical_tags import tags
@@ -15,6 +14,8 @@
     NeuroBlueprintError,
 )
 
+from .. import test_utils
+
 
 class TestLogging:
     @pytest.fixture(scope="function")

tests/tests_integration/test_settings.py

@@ -2,13 +2,14 @@
 import shutil
 
 import pytest
-import test_utils
-from base import BaseTest
 
 from datashuttle.configs import canonical_configs
 from datashuttle.utils import validation
 from datashuttle.utils.custom_exceptions import NeuroBlueprintError
 
+from .. import test_utils
+from ..base import BaseTest
+
 
 class TestPersistentSettings(BaseTest):
     @pytest.mark.parametrize("project", ["local", "full"], indirect=True)