Merge branch 'main' into Improve_umdp3_checks_I

Author: t00sa

github_scripts/get_git_sources.py

@@ -5,8 +5,13 @@
 # -----------------------------------------------------------------------------
 """
 Helper functions for cloning git sources in command line builds
+
+Reads CONFLICT_IGNORES environment variable to get a list of comma-separated
+files/directories to ignore merge conflicts in. These should be relative to the
+top-level of the repo. If not set, then a default list is defined below.
 """
 
+import os
 import re
 import subprocess
 from datetime import datetime
@@ -231,17 +236,24 @@ def handle_merge_conflicts(source: str, ref: str, loc: Path, dependency: str) ->
     If merge conflicts are in `rose-stem/` or `dependencies.yaml` then accept the
     current changes and mark as resolved.
     If others remain then raise an error
+    If CONFLICT_IGNORES environment variable is set, use that as a comma-separated list
+    of files/directories to ignore. If not set, use a default list below.
     """
 
     # For suites, merge conflicts in these files/directories are unimportant so accept
     # the current changes
-    for filepath in ("dependencies.yaml", "rose-stem"):
+    ignores = os.environ.get(
+        "CONFLICT_IGNORES", "dependencies.yaml,rose-stem,CONTRIBUTORS.md"
+    ).split(",")
+    need_commit = False
+    for filepath in ignores:
         full_path = loc / filepath
         if not full_path.exists():
             continue
         logger.warning(f"Ignoring merge conflicts in {filepath}")
         run_command(f"git -C {loc} checkout --ours -- {filepath}")
-        run_command(f"git -C {loc} add {filepath}")
+        run_command(f"git -C {loc} add -f {filepath}")
+        need_commit = True
 
     # Check if there are any remaining merge conflicts
     unmerged = get_unmerged(loc)
@@ -254,6 +266,10 @@ def handle_merge_conflicts(source: str, ref: str, loc: Path, dependency: str) ->
             "\n\nThese will need changing in the source branches to be useable together"
         )
 
+    # Need to commit the ignored conflict fixes
+    if need_commit:
+        run_command(f"git -C {loc} commit -m 'fix conflict'")
+
 
 def get_unmerged(loc: Path) -> list[str]:
     """

lfric_macros/apply_macros.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 ##############################################################################
 # (c) Crown copyright 2024 Met Office. All rights reserved.
 # The file LICENCE, distributed with this code, contains details of the terms
@@ -249,9 +249,10 @@ def __init__(
         self,
         tag: str,
         cname: str | None,
-        version: str,
+        version: str | None,
         apps: Path,
         core: Path,
+        jules: Path | None = None,
         testing: bool = False,
     ) -> None:
         self.tag: str = tag
@@ -267,7 +268,8 @@ def __init__(
             self.root_path: Path = apps
         else:
             self.root_path: Path = get_root_path(apps)
-        self.core_source: Path = self.get_dependency_paths(core)
+        self.core_source: Path = self.get_dependency_paths(core, "lfric_core")
+        self.jules_source: Path = self.get_dependency_paths(jules, "jules")
         self.set_rose_meta_path()
         if version is None:
             self.version: str = re.search(r".*vn(\d+\.\d+)(_.*)?", tag).group(1)
@@ -286,13 +288,15 @@ def __init__(
 
     def set_rose_meta_path(self) -> None:
         """
-        Set up the ROSE_META_PATH environment variable in order to use the Core
-        metadata. We also add the clone root path as this should allow the script to be
-        run from anywhere.
+        Set up the ROSE_META_PATH environment variable in order to use the Core and
+        Jules metadata. We also add the clone root path as this should allow the script
+        to be run from anywhere.
         Edit 02/2026 - remove backwards compatibility support for pre central-metadata
         """
         rose_meta_path: str = (
-            f"{self.root_path / 'rose-meta'}:{self.core_source / 'rose-meta'}"
+            f"{self.root_path / 'rose-meta'}:"
+            f"{self.core_source / 'rose-meta'}:"
+            f"{self.jules_source / 'rose-meta'}"
         )
         os.environ["ROSE_META_PATH"] = rose_meta_path
 
@@ -321,24 +325,23 @@ def parse_application_section(self, meta_dir: Path) -> Path:
     # Get Working Copy Functions
     ############################################################################
 
-    def get_dependency_paths(self, source: str | None) -> Path:
+    def get_dependency_paths(self, source: str | None, repo: str) -> Path:
         """
-        Parse the core command line arguments to get the path to a git clone.
+        Parse the core/jules command line arguments to get the path to a git clone.
         If the source isn't defined, first populate the source by reading the
         dependencies.yaml file.
         If the source is a remote GitHub source clone it to a temporary location
         Inputs:
             - source, The command line argument for the source. If not set this will be
               None
+            - repo, The name of the repository to clone
         Outputs:
             - The path to the source working copy to use
         """
 
-        repo = "lfric_core"
-
         # If source is None then read the dependencies.yaml file for the source
         if source is None:
-            source, ref = self.read_dependencies()
+            source, ref = self.read_dependencies(repo)
         if ":" in str(source):
             source_path = Path(source.split(":")[1]).expanduser()
         else:
@@ -361,7 +364,7 @@ def get_dependency_paths(self, source: str | None) -> Path:
         source = self.git_clone_temp(source, ref, repo)
         return source
 
-    def read_dependencies(self, repo: str = "lfric_core") -> tuple[str, str]:
+    def read_dependencies(self, repo: str) -> tuple[str, str]:
         """
         Read through the dependencies.yaml file for the source of the repo defined
         by repo. Uses self.root_path to locate the dependencies.yaml file.
@@ -1033,14 +1036,10 @@ def preprocess_macros(self, nproc: int) -> None:
                 if exception is not None:
                     executor.shutdown(wait=False, cancel_futures=True)
                     raise exception
-                print(
-                    "[INFO] Processed macro successfully written to "
-                    f"{
-                        self.parse_application_section(
-                            meta_order[write_tasks.index(task)]
-                        )
-                    }"
+                print_val = self.parse_application_section(
+                    meta_order[write_tasks.index(task)]
                 )
+                print(f"[INFO] Processed macro successfully written to {print_val}")
 
     ############################################################################
     # Upgrade Apps Functions
@@ -1265,7 +1264,15 @@ def parse_args() -> argparse.Namespace:
         "--core",
         default=None,
         help="The LFRic Core source being used."
-        "Either a path to a working copy or a git source."
+        "Either a path to a local clone or a github source."
+        "If not set, will be read from the dependencies.yaml",
+    )
+    parser.add_argument(
+        "-j",
+        "--jules",
+        default=None,
+        help="The Jules source being used."
+        "Either a path to a local clone or a github source."
         "If not set, will be read from the dependencies.yaml",
     )
     parser.add_argument(
@@ -1280,14 +1287,15 @@ def apply_macros_main(
     version: str | None = None,
     apps: Path = Path(".").absolute(),
     core: str | None = None,
+    jules: str | None = None,
 ) -> None:
     """
     Main function for this program
     """
 
     check_environment()
 
-    macro_object: ApplyMacros = ApplyMacros(tag, cname, version, apps, core)
+    macro_object: ApplyMacros = ApplyMacros(tag, cname, version, apps, core, jules)
 
     # Pre-process macros
     banner_print("Pre-Processing Macros")
@@ -1316,4 +1324,6 @@ def apply_macros_main(
 
 if __name__ == "__main__":
     args = parse_args()
-    apply_macros_main(args.tag, args.cname, args.version, args.apps, args.core)
+    apply_macros_main(
+        args.tag, args.cname, args.version, args.apps, args.core, args.jules
+    )

lfric_macros/check_macro_chains.py

@@ -135,8 +135,11 @@ def main() -> None:
 
     source_apps = Path(os.environ["SOURCE_ROOT"]) / "lfric_apps"
     source_core = Path(os.environ["SOURCE_ROOT"]) / "lfric_core"
+    source_jules = Path(os.environ["SOURCE_ROOT"]) / "jules"
 
-    macro_object = ApplyMacros("vn0.0_t0", None, "vn0.0", source_apps, source_core)
+    macro_object = ApplyMacros(
+        "vn0.0_t0", None, "vn0.0", source_apps, source_core, source_jules
+    )
     apps_meta_dirs = macro_object.find_meta_dirs(macro_object.root_path / "rose-meta")
 
     rose_apps = find_upgradeable_apps(

lfric_macros/release_lfric.py

@@ -241,7 +241,9 @@ def copy_head_meta(meta_dirs: list[Path], apps: Path, core: Path, version: str)
         _ = run_command(command)
 
 
-def update_meta_import_path(meta_dirs: list[Path], version: str) -> None:
+def update_meta_import_path(
+    meta_dirs: list[Path], version: str, jules_version: str
+) -> None:
     """
     Change HEAD to vnX.Y in meta import statements in the newly created
     vnX.Y/rose-meta.conf files
@@ -261,7 +263,10 @@ def update_meta_import_path(meta_dirs: list[Path], version: str) -> None:
             elif in_imports and not line.strip().startswith("="):
                 break
             if in_imports:
-                line = line.replace("HEAD", version)
+                if "jules-lfric" in line:
+                    line = line.replace("HEAD", jules_version)
+                else:
+                    line = line.replace("HEAD", version)
                 lines[i] = line
 
         with open(meta_file, "w") as f:
@@ -412,6 +417,13 @@ def parse_args() -> argparse.Namespace:
         type=version_number,
         help="The new version number we are updating to (format X.Y)",
     )
+    parser.add_argument(
+        "-j",
+        "--jules_version",
+        required=True,
+        help="The newly released version of Jules for jules-lfric metadata imports "
+        "(format X.Y)",
+    )
     parser.add_argument(
         "-t",
         "--ticket",
@@ -440,6 +452,7 @@ def parse_args() -> argparse.Namespace:
     args.core = args.core.expanduser().absolute()
     args.version = f"vn{args.version}"
     args.old_version = f"vn{args.old_version}"
+    args.jules_version = f"vn{args.jules_version}"
 
     return args
 
@@ -457,29 +470,16 @@ def main() -> None:
 
     set_dependency_path(args.apps, args.core)
 
-    # Find all metadata directories, excluing jules shared and lfric inputs as these
-    # have metadata but no macros.
+    # Find all metadata directories, excluing lfric-inputs as this has metadata but no
+    # macros.
     exclude_dirs = (
-        ".svn",
+        ".git",
         "rose-stem",
         "integration-test",
-        "lfric-jules-shared",
         "lfricinputs",
     )
     meta_dirs = find_meta_dirs([args.apps, args.core], exclude_dirs)
 
-    # Find JULES shared metadata directories and combine with all other metadirs for
-    # where they are handled differently
-    jules_meta_path = (
-        args.apps
-        / "interfaces"
-        / "jules_interface"
-        / "rose-meta"
-        / "lfric-jules-shared"
-    )
-    jules_shared_meta_dirs = find_meta_dirs([jules_meta_path])
-    meta_dirs_plus_jules = meta_dirs.union(jules_shared_meta_dirs)
-
     update_version_number(args.apps, args.version)
 
     update_variables_files(args.apps)
@@ -498,9 +498,9 @@ def main() -> None:
     )
     print("\n[INFO] Successfully upgraded apps")
 
-    copy_head_meta(meta_dirs_plus_jules, args.apps, args.core, args.version)
+    copy_head_meta(meta_dirs, args.apps, args.core, args.version)
 
-    update_meta_import_path(meta_dirs, args.version)
+    update_meta_import_path(meta_dirs, args.version, args.jules_version)
 
     upgrade_file_name = copy_versions_files(
         meta_dirs, args.old_version, args.version, args.apps, args.core

lfric_macros/tests/test_apply_macros.py

@@ -70,7 +70,15 @@ def __repr__(self):
 
 # Create an instance of the apply_macros class
 # Use /tmp for Core and Jules as these are not required for testing
-applymacros = ApplyMacros("vn0.0_t001", None, None, TEST_APPS_DIR, Path("/tmp"), True)
+applymacros = ApplyMacros(
+    tag="vn0.0_t001",
+    cname=None,
+    version=None,
+    apps=TEST_APPS_DIR,
+    core=Path("/tmp"),
+    jules=Path("/tmp"),
+    testing=True,
+)
 
 
 def test_read_versions_file():

lfric_macros/validate_rose_meta.py

@@ -20,7 +20,6 @@
 INVALID_METADATA = [
     "jules-lfric",
     "jules-lsm",
-    "lfric-jules-shared",
     "socrates-radiation",
     "um-aerosol",
     "um-boundary_layer",
@@ -191,6 +190,13 @@ def parse_args() -> argparse.Namespace:
         "required. If both are provided then it will be assumed that Apps is the "
         "repository to check.",
     )
+    parser.add_argument(
+        "-j",
+        "--jules",
+        default=None,
+        help="The path to the JULES source required for jules-lfric & jules-shared "
+        "metadata.",
+    )
 
     args = parser.parse_args()
 
@@ -204,6 +210,8 @@ def parse_args() -> argparse.Namespace:
         args.apps = Path(args.apps).absolute().expanduser()
     if args.core:
         args.core = Path(args.core).absolute().expanduser()
+    if args.jules:
+        args.jules = Path(args.jules).absolute().expanduser()
 
     return args
 
@@ -229,6 +237,9 @@ def main() -> None:
             # Apps hasn't been set
             source_path = args.core
             rose_meta_path = f"{args.core / 'rose-meta'}"
+    if args.jules:
+        meta_paths += f"-M {args.jules / 'rose-meta'} "
+        rose_meta_path += f":{args.jules / 'rose-meta'}"
 
     if check_rose_metadata(rose_meta_path, source_path) or check_rose_stem_apps(
         meta_paths, source_path