Merge branch 'main' into make_nightly_infrastructure_git

Author: james-bruten-mo

git-migration/config.json

@@ -179,7 +179,8 @@
         { "vn7.6": 28692 },
         { "vn7.7": 29181 },
         { "vn7.8": 29791 },
-        { "vn7.9": 30414 }
+        { "vn7.9": 30414 },
+        { "git_migration": 31238 }
       ]
     },
     {
@@ -242,7 +243,8 @@
         { "um13.6": 4075 },
         { "um13.7": 4898 },
         { "um13.8": 5509 },
-        { "um13.9": 6356 }
+        { "um13.9": 6356 },
+        { "git_migration": 7497 }
       ]
     },
     {
@@ -346,7 +348,8 @@
         { "vn1.22": 118501 },
         { "vn1.23": 122041 },
         { "vn1.24": 126461 },
-        { "vn1.25": 130034 }
+        { "vn1.25": 130034 },
+        { "git_migration": 131096 }
       ]
     },
     {
@@ -394,7 +397,7 @@
       "name": "um_meta",
       "description": "Metadata for the Met Office Unified Model (UM)",
       "trunk": "um/meta",
-      "tags": []
+      "tags": [{ "git_migration": 131163 }]
     },
     {
       "name": "um",
@@ -483,7 +486,8 @@
         { "vn8.1.1": 1226 },
         { "vn8.2": 1251 },
         { "vn8.3": 1288 },
-        { "vn8.4": 1386 }
+        { "vn8.4": 1386 },
+        { "git_migration": 1428 }
       ]
     },
     {

bdiff/__init__.py github_scripts/__init__.pybdiff/__init__.py renamed to github_scripts/__init__.py

@@ -0,0 +1,124 @@
+# *****************************COPYRIGHT*******************************
+# (C) Crown copyright Met Office. All rights reserved.
+# For further details please refer to the file COPYRIGHT.txt
+# which you should have received as part of this distribution.
+# *****************************COPYRIGHT*******************************
+"""
+Clone sources for a rose-stem run for use with git bdiff module in scripts
+"""
+
+import re
+import subprocess
+from typing import Optional
+from pathlib import Path
+from shutil import rmtree
+
+
+def run_command(
+    command: str, rval: bool = False
+) -> Optional[subprocess.CompletedProcess]:
+    """
+    Run a subprocess command and return the result object
+    Inputs:
+        - command, str with command to run
+    Outputs:
+        - result object from subprocess.run
+    """
+    command = command.split()
+    result = subprocess.run(
+        command,
+        capture_output=True,
+        text=True,
+        timeout=300,
+        shell=False,
+        check=False,
+    )
+    if result.returncode:
+        print(result.stdout, end="\n\n\n")
+        raise RuntimeError(
+            f"[FAIL] Issue found running command {command}\n\n{result.stderr}"
+        )
+    if rval:
+        return result
+
+
+def clone_repo_mirror(
+    source: str, repo_ref: str, parent: str, mirror_loc: Path, loc: Path
+) -> None:
+    """
+    Clone a repo source using a local git mirror.
+    Assume the mirror is set up as per the Met Office
+    """
+
+    # Remove if this clone already exists
+    if loc.exists():
+        rmtree(loc)
+
+    command = f"git clone {mirror_loc} {loc}"
+    run_command(command)
+
+    # If not provided a ref, return
+    if not repo_ref:
+        return
+
+    source = source.removeprefix("git@github.com:")
+    user = source.split("/")[0]
+    # Check that the user is different to the Upstream User
+    if user in parent.split("/")[0]:
+        user = None
+
+    # If the ref is a hash then we don't need the fork user as part of the fetch.
+    # Equally, if the user is the Upstream User, it's not needed
+    if not user or re.match(r"^\s*([0-9a-f]{40})\s*$", repo_ref):
+        fetch = repo_ref
+    else:
+        fetch = f"{user}/{repo_ref}"
+    commands = (
+        f"git -C {loc} fetch origin {fetch}",
+        f"git -C {loc} checkout FETCH_HEAD",
+    )
+    for command in commands:
+        run_command(command)
+
+
+def clone_repo(repo_source: str, repo_ref: str, loc: Path) -> None:
+    """
+    Clone the repo and checkout the provided ref
+    Only if a remote source
+    """
+
+    # Remove if this clone already exists
+    if loc.exists():
+        rmtree(loc)
+
+    # Create a clean clone location
+    loc.mkdir(parents=True)
+
+    commands = (
+        f"git -C {loc} init",
+        f"git -C {loc} remote add origin {repo_source}",
+        f"git -C {loc} fetch origin {repo_ref}",
+        f"git -C {loc} checkout FETCH_HEAD",
+    )
+    for command in commands:
+        run_command(command)
+
+
+def sync_repo(repo_source: str, repo_ref: str, loc: Path) -> None:
+    """
+    Rsync a local git clone and checkout the provided ref
+    """
+
+    # Remove if this clone already exists
+    if loc.exists():
+        rmtree(loc)
+
+    # Create a clean clone location
+    loc.mkdir(parents=True)
+
+    # Trailing slash required for rsync
+    command = f"rsync -av {repo_source}/ {loc}"
+    run_command(command)
+    if repo_ref:
+        command = f"git -C {loc} checkout {repo_ref}"
+        run_command(command)

github_scripts/get_git_sources.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# *****************************COPYRIGHT*******************************
+# (C) Crown copyright Met Office. All rights reserved.
+# For further details please refer to the file COPYRIGHT.txt
+# which you should have received as part of this distribution.
+# *****************************COPYRIGHT*******************************
+"""
+Clone sources for a rose-stem run for use with git bdiff module in scripts
+Only intended for use with rose-stem suites that have provided appropriate environment
+variables
+"""
+
+import os
+from datetime import datetime
+from pathlib import Path
+from ast import literal_eval
+from get_git_sources import clone_repo, clone_repo_mirror, sync_repo
+from typing import Dict
+
+
+def set_https(dependencies: Dict) -> Dict:
+    """
+    Change sources in a dependencies dictions to use https instead of ssh
+    """
+
+    print("Modifying Dependencies")
+    for dependency, values in dependencies.items():
+        if values["source"].startswith("git@github.com:"):
+            source = dependencies[dependency]["source"]
+            dependencies[dependency]["source"] = source.replace(
+                "git@github.com:", "https://github.com/"
+            )
+
+    return dependencies
+
+
+def main() -> None:
+
+    clone_loc = Path(os.environ["SOURCE_DIRECTORY"])
+
+    dependencies: Dict = literal_eval(os.environ["DEPENDENCIES"])
+
+    if os.environ.get("USE_TOKENS", "False") == "True":
+        dependencies = set_https(dependencies)
+
+    for dependency, values in dependencies.items():
+
+        print(
+            f"Extracting {dependency} at time {datetime.now()} "
+            f"using source {values['source']} and ref {values['ref']}"
+        )
+
+        loc = clone_loc / dependency
+
+        if ".git" in values["source"]:
+            if os.environ.get("USE_MIRRORS", "False") == "True":
+                mirror_loc = Path(os.environ["GIT_MIRROR_LOC"]) / values["parent"]
+                clone_repo_mirror(
+                    values["source"], values["ref"], values["parent"], mirror_loc, loc
+                )
+            else:
+                clone_repo(values["source"], values["ref"], loc)
+        else:
+            sync_repo(values["source"], values["ref"], loc)
+
+
+if __name__ == "__main__":
+    main()

bdiff/git_bdiff.py github_scripts/git_bdiff.pybdiff/git_bdiff.py renamed to github_scripts/git_bdiff.py

@@ -8,9 +8,6 @@
 Class containing helper methods for gathering data needed for a SuiteReport object
 """
 
-import sys
-
-sys.path.append("../")
 import re
 import shutil
 import sqlite3
@@ -19,18 +16,8 @@
 from collections import defaultdict
 from pathlib import Path
 from typing import Dict, List, Optional, Set, Union
-
-try:
-    from bdiff.git_bdiff import GitBDiff, GitInfo
-except ImportError:
-    try:
-        from git_bdiff import GitBDiff, GitInfo
-    except ImportError as err:
-        raise ImportError(
-            "Unable to import from git_bdiff module. This is included in the same "
-            "repository as this script and included with a relative import. Ensure "
-            "this script is being called from the correct place."
-        ) from err
+from git_bdiff import GitBDiff, GitInfo
+from get_git_sources import clone_repo, sync_repo
 
 
 class SuiteData:
@@ -217,20 +204,9 @@ def clone_sources(self) -> None:
         for dependency, data in self.dependencies.items():
             loc = self.temp_directory / dependency
             if data["source"].endswith(".git"):
-                commands = [
-                    f"git clone {data['source']} {loc}",
-                    f"git -C {loc} checkout {data['ref']}",
-                ]
-                for command in commands:
-                    self.run_command(command)
+                clone_repo(data["source"], data["ref"], loc)
             else:
-                source = data["source"]
-                if not source.endswith("/"):
-                    source = source + "/"
-                command = (
-                    f'rsync -e "ssh -o StrictHostKeyChecking=no" -avl {source} {loc}'
-                )
-                self.run_command(command, shell=True)
+                sync_repo(data["source"], data["ref"], loc)
 
     def determine_primary_source(self) -> str:
         """
@@ -298,21 +274,18 @@ def read_rose_conf(self) -> Dict[str, str]:
 
     def find_unknown_dependency(self, dependency: str) -> str:
         """
-        TEMPORARY
         The primary dependency may be unset in the dependencies file. In this case find
-        it from the *_SOURCE variable in the rose-suite.conf.
-        TODO: Once cylc provides the location of the source code itself, this method
-        should be changed to use that instead, as then the _SOURCE variable will be
-        removed
-        """
-
-        var = f"{dependency.upper()}_SOURCE".replace('"', "")
-        if var not in self.rose_data:
-            raise RuntimeError(f"Cant determine source for {dependency}")
-        rval = self.rose_data[var]
-        if "$ROSE_ORIG_HOST" in rval:
-            rval = rval.replace("$ROSE_ORIG_HOST", self.rose_data["ROSE_ORIG_HOST"])
-        return rval
+        it from the CYLC_WORKFLOW_SRC_DIR variable that gets set in the
+        flow-processed.cylc file
+        """
+
+        pattern = re.compile(rf"{dependency.upper()} SOURCE CLONE=(\S+)")
+        log_file = self.suite_path / "log" / "scheduler" / "log"
+        with open(log_file, "r") as f:
+            for line in f:
+                if match := pattern.search(line):
+                    return match.group(1).rstrip("/")
+        raise RuntimeError(f"Unable to find source for dependency {dependency}")
 
     def read_dependencies(self) -> Dict[str, Dict]:
         """

github_scripts/rose_stem_extract_source.py

@@ -373,8 +373,8 @@ def main() -> None:
 
     args = parse_args()
 
+    suite_report = SuiteReport(args.suite_path)
     try:
-        suite_report = SuiteReport(args.suite_path)
         suite_report.create_log()
         suite_report.write_log(args.log_path)
     finally:

suite_report_git/suite_data.py github_scripts/suite_data.pysuite_report_git/suite_data.py renamed to github_scripts/suite_data.py

@@ -18,6 +18,7 @@
 # dependency
 INVALID_METADATA = [
     "jules-lfric",
+    "jules-lsm",
     "lfric-jules-shared",
     "socrates-radiation",
     "um-aerosol",
@@ -217,16 +218,16 @@ def main():
     if args.apps:
         source_path = args.apps
         meta_paths += f"-M {os.path.join(args.apps, "rose-meta")} "
-        rose_meta_path += args.apps
+        rose_meta_path += f"{os.path.join(args.apps, "rose-meta")}"
     if args.core:
         meta_paths += f"-M {os.path.join(args.core, "rose-meta")} "
         if rose_meta_path:
             # Apps has already started this
-            rose_meta_path += f":{args.core}"
+            rose_meta_path += f":{os.path.join(args.core, "rose-meta")}"
         else:
             # Apps hasn't been set
             source_path = args.core
-            rose_meta_path = args.core
+            rose_meta_path = f"{os.path.join(args.core, "rose-meta")}"
 
     if check_rose_metadata(rose_meta_path, source_path) or check_rose_stem_apps(
         meta_paths, source_path