lots of changes, sort out.

Author: JoeZiminski

tests/conftest.py

@@ -0,0 +1,16 @@
+# Use a base image with the desired OS (e.g., Ubuntu, Debian, etc.)
+FROM ubuntu:latest
+# Install SSH server
+RUN apt-get update && \
+ apt-get install -y openssh-server
+RUN apt-get install nano
+# Create an SSH user
+RUN useradd -rm -d /home/sshuser -s /bin/bash -g root -G sudo -u 1000 sshuser
+# Set the SSH user's password (replace "password" with your desired password)
+RUN echo 'sshuser:password' | chpasswd
+# Allow SSH access
+RUN mkdir /var/run/sshd
+# Expose the SSH port
+EXPOSE 22
+# Start SSH server on container startup
+CMD ["/usr/sbin/sshd", "-D"]

tests/ssh_test_images/Dockerfile

@@ -55,3 +55,29 @@ def setup_hostkeys(project):
         project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True
     )
     restore_mock_input(orig_builtin)
+
+    orig_getpass = copy.deepcopy(ssh.getpass.getpass)
+    ssh.getpass.getpass = lambda _: "password"
+
+    ssh.setup_ssh_key(project.cfg, log=False)
+    ssh.getpass.getpass = orig_getpass
+
+
+def build_docker_image(project):
+    import os
+    import subprocess
+    from pathlib import Path
+
+    image_path = Path(__file__).parent / "ssh_test_images"
+    os.chdir(image_path)
+    subprocess.run("docker build -t ssh_server .", shell=True)
+    subprocess.run(
+        "docker run -d -p 22:22 ssh_server", shell=True
+    )  # ; docker build -t ssh_server .", shell=True)  # ;docker run -p 22:22 ssh_server
+
+    setup_project_for_ssh(
+        project,
+        central_path=f"/home/sshuser/datashuttle/{project.project_name}",
+        central_host_id="localhost",
+        central_host_username="sshuser",
+    )

tests/ssh_test_utils.py

@@ -1,34 +1,37 @@
 """
 """
 import copy
-import glob
 import shutil
-import time
+import stat
 from pathlib import Path
 
 import pandas as pd
+import paramiko
 import pytest
 import ssh_test_utils
 import test_utils
-from pytest import ssh_config
+
+# from pytest import ssh_config
 from test_file_conflicts_pathtable import get_pathtable
 
+from datashuttle.utils import ssh
+
+TEST_SSH = True  # TODO: base on whether docker / singularity is installed.
 
 class TestFileTransfer:
     @pytest.fixture(
         scope="class",
-        params=[  # Set running SSH or local filesystem (see docstring).
-            False,
+        params=[
+        #    False,
             pytest.param(
                 True,
                 marks=pytest.mark.skipif(
-                    ssh_config.TEST_SSH is False,
-                    reason="TEST_SSH is set to False.",
+                    TEST_SSH is False, reason="TEST_SSH is set to False."
                 ),
             ),
         ],
     )
-    def pathtable_and_project(self, request, tmpdir_factory):
+    def project_and_test_information(self, request, tmpdir_factory):
         """
         Create a project for SSH testing. Setup
         the project as normal, and switch configs
@@ -72,44 +75,24 @@ def pathtable_and_project(self, request, tmpdir_factory):
         testing_ssh = request.param
         tmp_path = tmpdir_factory.mktemp("test")
 
-        if testing_ssh:
-            base_path = ssh_config.FILESYSTEM_PATH
-            central_path = ssh_config.SERVER_PATH
-        else:
-            base_path = tmp_path / "test with space"
-            central_path = base_path
+        base_path = tmp_path / "test with space"
         test_project_name = "test_file_conflicts"
 
         project, cwd = test_utils.setup_project_fixture(
             base_path, test_project_name
         )
 
         if testing_ssh:
-            ssh_test_utils.setup_project_for_ssh(
-                project,
-                test_utils.make_test_path(
-                    central_path, test_project_name, "central"
-                ),
-                ssh_config.CENTRAL_HOST_ID,
-                ssh_config.USERNAME,
-            )
-
-            # Initialise the SSH connection
+            ssh_test_utils.build_docker_image(project)
             ssh_test_utils.setup_hostkeys(project)
-            shutil.copy(ssh_config.SSH_KEY_PATH, project.cfg.file_path.parent)
 
         pathtable = get_pathtable(project.cfg["local_path"])
         self.create_all_pathtable_files(pathtable)
-        project.testing_ssh = testing_ssh
 
-        yield [pathtable, project]
+        yield [pathtable, project, testing_ssh]
 
         test_utils.teardown_project(cwd, project)
 
-        if testing_ssh:
-            for result in glob.glob(ssh_config.FILESYSTEM_PATH):
-                shutil.rmtree(result)
-
     # -------------------------------------------------------------------------
     # Utils
     # -------------------------------------------------------------------------
@@ -156,14 +139,14 @@ def central_from_local(self, path_):
             ["histology", "behav", "all_ses_level_non_datatype"],
         ],
     )
-    @pytest.mark.parametrize("upload_or_download", ["upload", "download"])
+#    @pytest.mark.parametrize("upload_or_download", ["upload", "download"])
     def test_all_data_transfer_options(
         self,
-        pathtable_and_project,
+        project_and_test_information,
         sub_names,
         ses_names,
         datatype,
-        upload_or_download,
+#        upload_or_download,
     ):
         """
         Parse the arguments to filter the pathtable, getting
@@ -175,31 +158,31 @@ def test_all_data_transfer_options(
         on setting up and swapping local / central paths for
         upload / download tests.
         """
-        pathtable, project = pathtable_and_project
+        pathtable, project, testing_ssh = project_and_test_information
 
-        transfer_function = test_utils.handle_upload_or_download(
-            project,
-            upload_or_download,
-            swap_last_folder_only=project.testing_ssh,
-        )[0]
+  #      transfer_function = test_utils.handle_upload_or_download(
+   #         project,
+    #        upload_or_download,
+     #       swap_last_folder_only=testing_ssh,
+      #  )[0]
 
-        transfer_function(sub_names, ses_names, datatype, init_log=False)
+        project.upload(sub_names, ses_names, datatype, init_log=False)
+        # transfer_function(sub_names, ses_names, datatype, init_log=False)
 
-        if upload_or_download == "download":
-            test_utils.swap_local_and_central_paths(
-                project, swap_last_folder_only=project.testing_ssh
-            )
+       # if upload_or_download == "download":
+        #    test_utils.swap_local_and_central_paths(
+         #       project, swap_last_folder_only=testing_ssh
+          #  )
 
-        sub_names = self.parse_arguments(pathtable, sub_names, "sub")
-        ses_names = self.parse_arguments(pathtable, ses_names, "ses")
-        datatype = self.parse_arguments(pathtable, datatype, "datatype")
+        parsed_sub_names = self.parse_arguments(pathtable, sub_names, "sub")
+        parsed_ses_names = self.parse_arguments(pathtable, ses_names, "ses")
+        parsed_datatype = self.parse_arguments(pathtable, datatype, "datatype")
 
-        # Filter pathtable to get files that were expected
-        # to be transferred
+        # Filter pathtable to get files that were expected to be transferred
         (
             sub_ses_dtype_arguments,
             extra_arguments,
-        ) = self.make_pathtable_search_filter(sub_names, ses_names, datatype)
+        ) = self.make_pathtable_search_filter(parsed_sub_names, parsed_ses_names, parsed_datatype)
 
         datatype_folders = self.query_table(pathtable, sub_ses_dtype_arguments)
         extra_folders = self.query_table(pathtable, extra_arguments)
@@ -214,28 +197,90 @@ def test_all_data_transfer_options(
 
         # When transferring with SSH, there is a delay before
         # filesystem catches up
-        if project.testing_ssh:
-            time.sleep(0.5)
+      #  if testing_ssh:
+       #     time.sleep(0.5)
 
         # Check what paths were actually moved
         # (through the local filesystem), and test
-        path_to_search = (
-            self.central_from_local(project.cfg["local_path"]) / "rawdata"
-        )
-        all_transferred = path_to_search.glob("**/*")
-        paths_to_transferred_files = list(
-            filter(Path.is_file, all_transferred)
-        )
+        def sftp_recursive_search(sftp, path_, all_filenames):
+            try:
+                sftp.stat(path_)
+            except FileNotFoundError:
+                return
+
+            for file_or_folder in sftp.listdir_attr(path_):
+                if stat.S_ISDIR(file_or_folder.st_mode):
+                    sftp_recursive_search(
+                        sftp,
+                        path_ + "/" + file_or_folder.filename,
+                        all_filenames,
+                    )
+                else:
+                    all_filenames.append(path_ + "/" + file_or_folder.filename)
+
+        with paramiko.SSHClient() as client:
+            ssh.connect_client(client, project.cfg)
+
+            sftp = client.open_sftp()
+
+            all_filenames = []
+
+            sftp_recursive_search(
+                sftp,
+                (project.cfg["central_path"] / "rawdata").as_posix(),
+                all_filenames,
+            )
 
-        assert sorted(paths_to_transferred_files) == sorted(
-            expected_transferred_paths
-        )
+            paths_to_transferred_files = []
+            for path_ in all_filenames:
+                parts = Path(path_).parts
+                paths_to_transferred_files.append(
+                    Path(*parts[parts.index("rawdata") :])
+                )
+
+            expected_transferred_paths_ = []
+            for path_ in expected_transferred_paths:
+                parts = Path(path_).parts
+                expected_transferred_paths_.append(
+                    Path(*parts[parts.index("rawdata") :])
+                )
+
+            assert sorted(paths_to_transferred_files) == sorted(
+                expected_transferred_paths_
+            )
+
+        project.upload_all()
+        shutil.rmtree(project.cfg["local_path"] / "rawdata")  # TOOD: var
+
+        breakpoint()
+
+        true_local_path = project.cfg["local_path"]
+        tmp_local_path = project.cfg["local_path"] / "tmp_local"
+        tmp_local_path.mkdirs()
+        project.update_config("local_path", tmp_local_path)
+
+        project.download(sub_names, ses_names, datatype, init_log=False)  # TODO: why is this connecting so many times?
+
+        all_transferred = list((project.cfg["local_path"] / "rawdata").glob("**/*"))
+        all_transferred = [path_ for path_ in all_transferred if path_.is_file()]
+
+        paths_to_transferred_files = []
+        for path_ in all_transferred:  # TODO: rename all filenames
+            parts = Path(path_).parts
+            paths_to_transferred_files.append(
+                Path(*parts[parts.index("rawdata"):])
+            )
+
+        assert sorted(paths_to_transferred_files) == sorted(expected_transferred_paths_)
+
+        shutil.rmtree(project.cfg["local_path"])  # TOOD: var
+
+        project.update_config("local_path", true_local_path)
+
+        with paramiko.SSHClient() as client:
+            ssh.connect_client(client, project.cfg)
 
-        # Teardown here, because we have session scope.
-        try:
-            shutil.rmtree(self.central_from_local(project.cfg["local_path"]))
-        except FileNotFoundError:
-            pass
+            client.exec_command(f"rm -rf {(project.cfg['central_path'] / 'rawdata').as_posix()}")  # TODO: own function as need to do on teardown)
 
     # ---------------------------------------------------------------------------------------------------------------
     # Utils

tests/tests_integration/test_ssh_file_transfer.py

@@ -6,12 +6,14 @@
 import pytest
 import ssh_test_utils
 import test_utils
-from pytest import ssh_config
 
+# from pytest import ssh_config
 from datashuttle.utils import ssh
 
+TEST_SSH = False
 
-@pytest.mark.skipif(ssh_config.TEST_SSH is False, reason="TEST_SSH is false")
+
+@pytest.mark.skipif(TEST_SSH is False, reason="TEST_SSH is false")
 class TestSSH:
     @pytest.fixture(scope="function")
     def project(test, tmp_path):