Tidy ups.

Author: JoeZiminski

tests/ssh_test_utils.py

@@ -1,7 +1,13 @@
 import builtins
 import copy
+import platform
 import stat
-
+import subprocess
+import sys
+import warnings
+import os
+import subprocess
+from pathlib import Path
 import paramiko
 
 from datashuttle.utils import rclone, ssh
@@ -49,33 +55,54 @@ def restore_mock_input(orig_builtin):
     builtins.input = orig_builtin
 
 
-def setup_hostkeys(project):
+def setup_hostkeys(project, setup_ssh_key_pair=True):  # TODO: RENAME FUNCTION
     """
     Convenience function to verify the server hostkey.
+
+    This requires monkeypatching a number of functions involved
+    in the SSH setup process. `input()` is patched to always
+    return the required hostkey confirmation "y". `getpass()` is
+    patched to allways return the password for the container in which
+    SSH tests are run. `isatty()` is patched because when running this
+    for some reason it appears to be in a TTY - this might be a
+    container thing.
     """
+    # Monkeypatch
     orig_builtin = setup_mock_input(input_="y")
-    ssh.verify_ssh_central_host(
-        project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True
-    )
-    restore_mock_input(orig_builtin)
 
     orig_getpass = copy.deepcopy(ssh.getpass.getpass)
     ssh.getpass.getpass = lambda _: "password"  # type: ignore
 
-    ssh.setup_ssh_key(project.cfg, log=False)
+    orig_isatty = copy.deepcopy(sys.stdin.isatty)
+    sys.stdin.isatty = lambda: True
+
+    # Run setup
+    verified = ssh.verify_ssh_central_host(
+        project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True
+    )
+
+    if setup_ssh_key_pair:
+        ssh.setup_ssh_key(project.cfg, log=False)
+
+    # Restore functions
+    restore_mock_input(orig_builtin)
     ssh.getpass.getpass = orig_getpass
+    sys.stdin.isatty = orig_isatty
+
+    return verified
 
 
 def build_docker_image(project):
-    import os
-    import subprocess
-    from pathlib import Path
+    """"""
+    container_software = is_docker_or_singularity_installed()
+    assert container_software is not False, ("docker or singularity not installed, "
+                                             "this should be checked at the top of test script")
 
     image_path = Path(__file__).parent / "ssh_test_images"
     os.chdir(image_path)
-    subprocess.run("docker build -t ssh_server .", shell=True)
+    subprocess.run(f"{container_software} build -t ssh_server .", shell=True)
     subprocess.run(
-        "docker run -d -p 22:22 ssh_server", shell=True
+        f"{container_software} run -d -p 22:22 ssh_server", shell=True
     )  # ; docker build -t ssh_server .", shell=True)  # ;docker run -p 22:22 ssh_server
 
     setup_project_for_ssh(
@@ -118,3 +145,32 @@ def recursive_search_central(project):
             all_filenames,
         )
     return all_filenames
+
+
+def get_test_ssh():
+    """"""
+    if is_docker_or_singularity_installed():
+        test_ssh = True
+    else:
+        warnings.warn("SSH tests are not run as docker (Windows, macOS) "
+                      "or singularity (Linux) is not installed.")
+        test_ssh = False
+
+    return test_ssh
+
+
+def is_docker_or_singularity_installed():  # TODO: need to test
+    """"""
+    check_install = lambda command: subprocess.run(
+        command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+    ).returncode == 0
+
+    installed = False
+    if platform.system() == "Linux":
+        if check_install("singularity version"):
+            installed = "singularity"
+    else:
+        if check_install("docker -v"):
+            installed = "docker"
+
+    return installed

tests/tests_integration/test_ssh_file_transfer.py

@@ -11,10 +11,9 @@
 import test_utils
 from file_conflicts_pathtable import get_pathtable
 
-# from pytest import ssh_config
 from datashuttle.utils import ssh
 
-TEST_SSH = True  # TODO: base on whether docker / singularity is installed.
+TEST_SSH = ssh_test_utils.get_test_ssh()
 
 
 PARAM_SUBS = [
@@ -142,24 +141,28 @@ def test_combinations_filesystem_transfer(
         except FileNotFoundError:
             pass
 
-    @pytest.mark.parametrize("sub_names", PARAM_SUBS)
-    @pytest.mark.parametrize("ses_names", PARAM_SES)
-    @pytest.mark.parametrize("datatype", PARAM_DATATYPE)
+    @pytest.mark.skipif("not TEST_SSH", reason="TEST_SSH is false")
+    @pytest.mark.parametrize("sub_names", [["all"], ["all_non_sub", "sub-002"]])
+    @pytest.mark.parametrize("ses_names", [["all"], ["ses-002_random-key"], ["all_non_ses"]])
+    @pytest.mark.parametrize("datatype", [["all"], ["anat", "all_ses_level_non_datatype"]])
     def test_combinations_ssh_transfer(
         self,
         ssh_setup,
         sub_names,
         ses_names,
         datatype,
     ):
-        """ """
+        """
+        This is very slow. maybe 8 s per test. Nearly all in the
+        upload() and download() part so unavoidable. Most code is shared,
+        this should be okay though my paranoid
+        """
         pathtable, project = ssh_setup
 
         true_central_path = project.cfg["central_path"]
-        tmp_central_path = project.cfg["central_path"] / "tmp"
+        tmp_central_path = project.cfg["central_path"] / "tmp" / project.project_name
         project.update_config("central_path", tmp_central_path)
 
-        breakpoint()
         project.upload(sub_names, ses_names, datatype, init_log=False)
 
         expected_transferred_paths = self.get_expected_transferred_paths(
@@ -184,17 +187,18 @@ def test_combinations_ssh_transfer(
             ssh.connect_client(client, project.cfg)
             client.exec_command(
                 f"rm -rf {(tmp_central_path).as_posix()}"
-            )  # TODO: own function as need to do on teardown)
+            )
 
         true_local_path = project.cfg["local_path"]
-        tmp_local_path = project.cfg["local_path"] / "tmp"
-        tmp_local_path.mkdir()
+        tmp_local_path = project.cfg["local_path"] / "tmp" / project.project_name
+        tmp_local_path.mkdir(exist_ok=True, parents=True)
+
         project.update_config("local_path", tmp_local_path)
         project.update_config("central_path", true_central_path)
 
         project.download(
             sub_names, ses_names, datatype, init_log=False
-        )  # TODO: why is this connecting so many times? [during search - make issue]
+        )
 
         all_transferred = list((tmp_local_path / "rawdata").glob("**/*"))
         all_transferred = [

tests/tests_integration/test_ssh_setup.py

@@ -1,19 +1,14 @@
-"""
-SSH configs are set in conftest.py . The password
-should be stored in a file called test_ssh_password.txt located
-in the same folder as test_ssh.py
-"""
 import pytest
 import ssh_test_utils
 import test_utils
 
 # from pytest import ssh_config
 from datashuttle.utils import ssh
 
-TEST_SSH = True
+TEST_SSH = ssh_test_utils.get_test_ssh()
 
 
-@pytest.mark.skipif(TEST_SSH is False, reason="TEST_SSH is false")
+@pytest.mark.skipif("not TEST_SSH", reason="TEST_SSH is false")
 class TestSSH:
     @pytest.fixture(scope="function")
     def project(test, tmp_path):
@@ -28,12 +23,7 @@ def project(test, tmp_path):
             tmp_path, test_project_name
         )
 
-        ssh_test_utils.setup_project_for_ssh(
-            project,
-            central_path=f"/home/sshuser/datashuttle/{project.project_name}",  # TODO: centralise these
-            central_host_id="localhost",
-            central_host_username="sshuser",
-        )
+        ssh_test_utils.build_docker_image(project)  # TODO: rename function
 
         yield project
         test_utils.teardown_project(cwd, project)
@@ -70,16 +60,14 @@ def test_verify_ssh_central_host_accept(self, capsys, project):
         and check hostkey is successfully accepted and written to configs.
         """
         test_utils.clear_capsys(capsys)
-        orig_builtin = ssh_test_utils.setup_mock_input(input_="y")
 
-        verified = ssh.verify_ssh_central_host(
-            project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True
+        verified = ssh_test_utils.setup_hostkeys(
+            project, setup_ssh_key_pair=False
         )
 
-        ssh_test_utils.restore_mock_input(orig_builtin)
-
         assert verified
         captured = capsys.readouterr()
+
         assert captured.out == "Host accepted.\n"
 
         with open(project.cfg.hostkeys_path, "r") as file: