Merge pull request #210 from fairagro/github_use_https

featuring git via https

Author: Zalfsten

.vscode/launch.json

@@ -4,14 +4,21 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-
         {
             "name": "Python Debugger: container-structure-test",
             "type": "debugpy",
             "request": "launch",
             "module": "middleware.main",
             "console": "integratedTerminal",
             "args": ["-c", "test/container-structure-test/image_test_config.yml", "--no-git"]
+        },
+        {
+            "name": "Python Debugger: test run",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "middleware.main",
+            "console": "integratedTerminal",
+            "args": ["-c", "config.yml"]
         }
     ]
 }

README.md

@@ -67,12 +67,11 @@ docker run \
 A few notes on this docker run call:
 
 - We run the container without root privileges as user `nonroot`. This user is defined in the Wolfi base image.
-- `config.yml` and `ssh_key.pem` specify the configuration. `config.yml` is mounted as volume, whereas the content of `ssh_key.epm`
+- `config.yml` and `ssh_key.pem` specify the configuration. `config.yml` is mounted as volume, whereas the content of `ssh_key.pem`
   is passed as environment variable `SSH_PRIVATE_KEY`. This is to prevent from permission issues. The ssh key is for git interaction with the
-  remote repository specified in the config file. Of course it needs to be already known to the git repo.
-- The local git working folder would preferably also be mounted into the container so its contents could be cached between container runs.
-  But this seems not to be possible -- at least not without administrational permission on the host machine. The issue is that mounted
-  volumes always belong to root, but the container does not run with root permissions, so it has no write access to the folder.
+  remote repository specified in the config file. Of course the key already needs to be registered with the git repo.
+- As an alternative to access git via `ssh`, it's also possible to use the `https` protocol. In this case you will need to create a
+  personal access token in the github user interface to gain write access to the middleware repo. This access token is passed via the environment variable `ACCESS_TOKEN`.
 
 ## Notes on the python version ##
 

config.yml

@@ -38,7 +38,7 @@ http_client:
 # You can omit the ssh_key_path if you have a local git setup with your personal key.
 # If branch is omitted, the main branch is used
 git:
-  repo_url: git@github.com:fairagro/middleware_repo.git
+  repo_url: https://github.com/fairagro/middleware_repo.git
   branch: main
   local_path: output
   user_name: "FAIRagro middleware"

middleware/git_repo/__init__.py

@@ -24,6 +24,7 @@
 from pathlib import Path, PurePosixPath
 import os
 import tempfile
+import weakref
 
 import git
 
@@ -33,7 +34,7 @@ class GitRepoConfig(NamedTuple):
     A class for the configuration of a GitRepo.
     """
 
-    repo_url: Annotated[str, "The ssh URL of the git repository"]
+    repo_url: Annotated[str, "The ssh or https URL of the git repository"]
     local_path: Annotated[str, "The local path of the git repository"]
     user_name: Annotated[str, "The name of git user"]
     user_email: Annotated[str, "The email address of git usere"]
@@ -72,11 +73,8 @@ class GitRepo:
 
     def __init__(self, config: GitRepoConfig) -> None:
         self._config = config
-        self._ssh_tempdir = tempfile.TemporaryDirectory() # pylint: disable=R1732
-        self._ssh_key = os.path.abspath(
-            os.path.join(self._ssh_tempdir.name, "ssh_key"))
-        self._ssh_authorized_keys = os.path.abspath(
-            os.path.join(self._ssh_tempdir.name, "authorized_keys"))
+        self._ssh_tempdir = None
+        self._finalizer = weakref.finalize(self, self.cleanup)
         self._repo = self._setup()
 
     # Make this class a context manager to reliably delete the temp dir
@@ -112,7 +110,15 @@ def __exit__(
         -------
             None
         """
-        self._ssh_tempdir.cleanup()
+        self.cleanup()
+
+    def cleanup(self) -> None:
+        """
+        delete temp dir, if used
+        """
+        if self._ssh_tempdir:
+            self._ssh_tempdir.cleanup()
+            self._ssh_tempdir = None
 
     @property
     def working_dir(self) -> Union[str, os.PathLike[str]]:
@@ -169,27 +175,26 @@ def push(self) -> Any:
         return self._repo.remotes.origin.push()
 
     @staticmethod
-    def _make_ssh_key_path(original_path):
+    def _make_ssh_key_path(path: Path) -> PurePosixPath:
         # This is some ugly workaround for git on Windows. In this case git is based on MSYS, so the
         # ssh command requires POSIX compatible paths, whereas otherwise we deal with Windows paths
         # on Windows. Thus we need to convert the Windows path to the ssh key to MSYS-POSIX.
         # Be aware: this is brittle as it assumes that we always use MSYS ssh on Windows. Maybe
         # there are other ways to setup git.
-        path = Path(original_path)
         parts = path.parts
         if parts[0].endswith(':\\'):
             parts = ['/', parts[0].rstrip(':\\'), *parts[1:]]
         return PurePosixPath(*parts)
 
-    def _setup(self):
-        # find out local repo path
-        local_path = self._config.local_path
+    def _setup_ssh_protocol(self) -> str:
+        # We need a temp dir to store some ssh specific files
+        self._ssh_tempdir = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with
 
         # find the ssh key and use it. We need an absolute path for this so git can find it.
         # no matter which is the current working directory.
-        ssh_key = GitRepo._make_ssh_key_path(self._ssh_key)
-        ssh_authorized_keys = GitRepo._make_ssh_key_path(
-            self._ssh_authorized_keys)
+        temp_dir = Path(self._ssh_tempdir.name)
+        ssh_key = GitRepo._make_ssh_key_path(temp_dir / "ssh_key")
+        ssh_authorized_keys = GitRepo._make_ssh_key_path(temp_dir / "ssh_authorized_keys")
 
         # Get key from env and write to file
         private_key = os.environ.get("SSH_PRIVATE_KEY")
@@ -214,14 +219,42 @@ def _setup(self):
             f'-o StrictHostKeyChecking=yes'
         )
 
+        return self._config.repo_url
+
+    def _setup_https_protocol(self) -> str:
+        access_token = os.environ.get("ACCESS_TOKEN")
+        if access_token:
+            _, rest = self._config.repo_url.split("https://", 1)
+            repo_url = f"https://{access_token}@{rest}"
+        else:
+            repo_url = self._config.repo_url
+
+        return repo_url
+
+
+    def _setup(self):
+        # find out local repo path
+        local_path = self._config.local_path
+
+        # detect git protocol and perform corrersponding setup
+        if self._config.repo_url.startswith('git@'):
+            repo_url = self._setup_ssh_protocol()
+        elif self._config.repo_url.startswith('https://'):
+            repo_url = self._setup_https_protocol()
+        else:
+            raise ValueError(
+                f"The specified git repo URL '{self._config.repo_url}' "
+                "does neither use the https nor the ssh protocol"
+            )
+
         # Initialize existing repo or clone it, if this hasn't been done yet
         try:
             repo = git.Repo(local_path)
-            if repo.remotes.origin.url != self._config.repo_url:
-                raise RuntimeError("Repository " + local_path + "already exists and is not a " +
-                                   " clone of " + self._config.repo_url)
+            if repo.remotes.origin.url != repo_url:
+                raise RuntimeError(f"Repository '{local_path}' already exists and is not a "
+                                   f"clone of '{self._config.repo_url}'")
         except (git.NoSuchPathError, git.InvalidGitRepositoryError):
-            repo = git.Repo.clone_from(self._config.repo_url, local_path)
+            repo = git.Repo.clone_from(repo_url, local_path)
 
         # set git config
         config_writer = repo.config_writer()