Add support for group-writable managed directories

This is needed for directory used by NCO to point to the
latest E3SM-Unified pixi environment.

Author: xylar

mache/deploy/run.py

@@ -1072,17 +1072,35 @@ def _apply_deploy_permissions(
 
     managed_paths = list(dict.fromkeys(managed_paths))
 
-    if not managed_paths:
-        return
+    if managed_paths:
+        update_permissions(
+            managed_paths,
+            group,
+            show_progress=True,
+            group_writable=False,
+            other_readable=world_readable,
+            recursive=True,
+        )
 
-    update_permissions(
-        managed_paths,
-        group,
-        show_progress=True,
-        group_writable=False,
-        other_readable=world_readable,
-        recursive=True,
-    )
+    group_writable_dirs = [
+        path
+        for path in (
+            _normalize_permission_path(path)
+            for path in shared_artifacts.group_writable_dirs
+        )
+        if path is not None
+    ]
+    group_writable_dirs = list(dict.fromkeys(group_writable_dirs))
+
+    if group_writable_dirs:
+        update_permissions(
+            group_writable_dirs,
+            group,
+            show_progress=True,
+            group_writable=True,
+            other_readable=world_readable,
+            recursive=False,
+        )
 
 
 def _normalize_permission_path(path: str | None) -> str | None:

mache/deploy/shared.py

@@ -12,6 +12,7 @@ class SharedDeployArtifacts:
     base_path: str | None = None
     managed_dirs: list[str] = field(default_factory=list)
     managed_files: list[str] = field(default_factory=list)
+    group_writable_dirs: list[str] = field(default_factory=list)
 
 
 def create_shared_deploy_artifacts(
@@ -29,7 +30,7 @@ def create_shared_deploy_artifacts(
         repo_root=repo_root,
         field_name='shared.base_path',
     )
-    managed_dirs = _normalize_path_entries(
+    managed_dirs, group_writable_dirs = _normalize_managed_directory_entries(
         shared_cfg.get('managed_directories'),
         repo_root=repo_root,
         field_name='shared.managed_directories',
@@ -81,9 +82,16 @@ def create_shared_deploy_artifacts(
         dest_link.symlink_to(target_path)
         managed_dirs.append(str(dest_link.parent))
 
+    managed_dirs = _dedupe_existing_paths(managed_dirs)
+    group_writable_dir_set = set(_dedupe_existing_paths(group_writable_dirs))
+    group_writable_dirs = [
+        path for path in managed_dirs if path in group_writable_dir_set
+    ]
+
     return SharedDeployArtifacts(
         base_path=base_path,
-        managed_dirs=_dedupe_existing_paths(managed_dirs),
+        managed_dirs=managed_dirs,
+        group_writable_dirs=group_writable_dirs,
         managed_files=_dedupe_existing_paths(managed_files),
     )
 
@@ -133,6 +141,52 @@ def _normalize_path_entries(
     return entries
 
 
+def _normalize_managed_directory_entries(
+    value: Any,
+    *,
+    repo_root: str,
+    field_name: str,
+) -> tuple[list[str], list[str]]:
+    if value is None:
+        return [], []
+    if not isinstance(value, list):
+        raise ValueError(f'{field_name} must be a list if provided')
+
+    entries: list[str] = []
+    group_writable_entries: list[str] = []
+    for index, item in enumerate(value):
+        item_field_name = f'{field_name}[{index}]'
+        path_value: Any
+        group_writable: bool
+        if isinstance(item, str):
+            path_value = item
+            group_writable = False
+        elif isinstance(item, dict):
+            path_value = item.get('path')
+            raw_group_writable = _coerce_optional_bool(
+                item.get('group_writable'),
+                field_name=f'{item_field_name}.group_writable',
+            )
+            group_writable = (
+                False if raw_group_writable is None else raw_group_writable
+            )
+        else:
+            raise ValueError(
+                f'{item_field_name} must be a string or mapping with a path'
+            )
+
+        path = _resolve_path(
+            value=path_value,
+            repo_root=repo_root,
+            field_name=f'{item_field_name}.path',
+        )
+        entries.append(path)
+        if group_writable:
+            group_writable_entries.append(path)
+
+    return entries, group_writable_entries
+
+
 def _normalize_load_script_copy_entries(
     value: Any,
     *,
@@ -215,6 +269,26 @@ def _normalize_load_script_symlink_entries(
     return list(deduped.values())
 
 
+def _coerce_optional_bool(
+    value: Any,
+    *,
+    field_name: str,
+) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        candidate = value.strip().lower()
+        if candidate in ('true', 'yes', 'on', '1'):
+            return True
+        if candidate in ('false', 'no', 'off', '0'):
+            return False
+        if candidate in ('', 'none', 'null', 'dynamic'):
+            return None
+    raise ValueError(f'{field_name} must be a boolean if provided')
+
+
 def _resolve_path(
     *,
     value: Any,

mache/deploy/templates/config.yaml.j2.j2

@@ -194,6 +194,12 @@ shared:
   # post-deploy permission update step. This is helpful when a downstream
   # pre_publish hook creates additional shared artifacts that mache itself
   # does not create.
+  # `managed_directories` entries may be either:
+  #   - "/absolute/or/relative/path/to/directory"
+  #   - {path: "/path/to/directory", group_writable: true}
+  #
+  # `group_writable: true` is applied only to that directory path itself,
+  # not recursively to its contents.
   managed_directories: []
   managed_files: []
 

mache/deploy/templates/hooks.py.j2

@@ -94,7 +94,8 @@ def pre_pixi(ctx: DeployContext) -> dict[str, Any] | None:
     # - runtime["shared"]["base_path"]: shared base directory to update
     #     permissions on recursively before path-specific updates
     # - runtime["shared"]["managed_directories"]: extra shared directories to
-    #     include in the permission update step
+    #     include in the permission update step. Entries may be strings or
+    #     mappings like {"path": "/shared/latest", "group_writable": True}.
     # - runtime["shared"]["managed_files"]: extra shared files to include in
     #     the permission update step
     #
@@ -114,6 +115,9 @@ def pre_pixi(ctx: DeployContext) -> dict[str, Any] | None:
     # updates.setdefault("shared", {})["load_script_copies"] = [
     #     "/shared/load_my_software.sh",
     # ]
+    # updates.setdefault("shared", {})["managed_directories"] = [
+    #     {"path": "/shared/latest", "group_writable": True},
+    # ]
 
     return updates
 

tests/test_deploy_run.py

@@ -1195,6 +1195,67 @@ def _fake_update_permissions(*args, **kwargs):
     assert third_kwargs['recursive'] is True
 
 
+def test_apply_deploy_permissions_applies_group_writable_overlay(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+):
+    prefix = tmp_path / 'prefix'
+    prefix.mkdir()
+
+    shared_base = tmp_path / 'shared'
+    shared_base.mkdir()
+    writable_dir = shared_base / 'latest'
+    writable_dir.mkdir()
+
+    calls = []
+
+    def _fake_update_permissions(*args, **kwargs):
+        calls.append((args, kwargs))
+
+    monkeypatch.setattr(
+        deploy_run, 'update_permissions', _fake_update_permissions
+    )
+
+    logger = deploy_run.logging.getLogger(
+        'test-apply-deploy-permissions-group-writable'
+    )
+    logger.handlers = [deploy_run.logging.NullHandler()]
+    logger.propagate = False
+
+    deploy_run._apply_deploy_permissions(
+        prefix=str(prefix),
+        extra_prefixes=None,
+        load_script_paths=[],
+        spack_paths=[],
+        shared_artifacts=SharedDeployArtifacts(
+            base_path=str(shared_base),
+            managed_dirs=[str(writable_dir)],
+            group_writable_dirs=[str(writable_dir)],
+            managed_files=[],
+        ),
+        group='e3sm',
+        world_readable=True,
+        logger=logger,
+    )
+
+    assert len(calls) == 3
+
+    first_args, first_kwargs = calls[0]
+    assert first_args == (str(shared_base), 'e3sm')
+    assert first_kwargs['group_writable'] is False
+    assert first_kwargs['recursive'] is True
+
+    second_args, second_kwargs = calls[1]
+    assert second_args == (str(prefix), 'e3sm')
+    assert second_kwargs['group_writable'] is False
+    assert second_kwargs['recursive'] is False
+
+    third_args, third_kwargs = calls[2]
+    assert third_args == ([str(writable_dir)], 'e3sm')
+    assert third_kwargs['group_writable'] is True
+    assert third_kwargs['other_readable'] is True
+    assert third_kwargs['recursive'] is False
+
+
 def test_apply_deploy_permissions_updates_shared_base_first(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ):

tests/test_deploy_shared.py

@@ -90,6 +90,133 @@ def test_create_shared_deploy_artifacts_resolves_runtime_base_path(
     )
 
 
+def test_create_shared_deploy_artifacts_marks_group_writable_dirs(
+    tmp_path: Path,
+):
+    repo_root = tmp_path / 'repo'
+    repo_root.mkdir()
+
+    readonly_dir = repo_root / 'shared' / 'readonly'
+    writable_dir = repo_root / 'shared' / 'writable'
+    duplicate_dir = repo_root / 'shared' / 'duplicate'
+    runtime_dir = repo_root / 'shared' / 'runtime'
+    for path in [readonly_dir, writable_dir, duplicate_dir, runtime_dir]:
+        path.mkdir(parents=True)
+
+    artifacts = create_shared_deploy_artifacts(
+        config={
+            'shared': {
+                'managed_directories': [
+                    'shared/readonly',
+                    {
+                        'path': 'shared/writable',
+                        'group_writable': 'true',
+                    },
+                    {
+                        'path': 'shared/duplicate',
+                        'group_writable': False,
+                    },
+                    {
+                        'path': 'shared/duplicate',
+                        'group_writable': True,
+                    },
+                ],
+            }
+        },
+        runtime={},
+        repo_root=str(repo_root),
+        load_script_paths=[],
+        logger=_logger(),
+    )
+
+    assert artifacts == SharedDeployArtifacts(
+        managed_dirs=[
+            str(readonly_dir),
+            str(writable_dir),
+            str(duplicate_dir),
+        ],
+        group_writable_dirs=[
+            str(writable_dir),
+            str(duplicate_dir),
+        ],
+    )
+
+    runtime_artifacts = create_shared_deploy_artifacts(
+        config={
+            'shared': {
+                'managed_directories': ['shared/readonly'],
+            }
+        },
+        runtime={
+            'shared': {
+                'managed_directories': [
+                    {
+                        'path': 'shared/runtime',
+                        'group_writable': True,
+                    }
+                ],
+            }
+        },
+        repo_root=str(repo_root),
+        load_script_paths=[],
+        logger=_logger(),
+    )
+
+    assert runtime_artifacts == SharedDeployArtifacts(
+        managed_dirs=[str(runtime_dir)],
+        group_writable_dirs=[str(runtime_dir)],
+    )
+
+
+def test_create_shared_deploy_artifacts_validates_managed_directory_entries(
+    tmp_path: Path,
+):
+    with pytest.raises(
+        ValueError,
+        match='shared.managed_directories\\[0\\] must be a string',
+    ):
+        create_shared_deploy_artifacts(
+            config={'shared': {'managed_directories': [42]}},
+            runtime={},
+            repo_root=str(tmp_path),
+            load_script_paths=[],
+            logger=_logger(),
+        )
+
+    with pytest.raises(
+        ValueError,
+        match='shared.managed_directories\\[0\\].path must not be null',
+    ):
+        create_shared_deploy_artifacts(
+            config={'shared': {'managed_directories': [{}]}},
+            runtime={},
+            repo_root=str(tmp_path),
+            load_script_paths=[],
+            logger=_logger(),
+        )
+
+    with pytest.raises(
+        ValueError,
+        match='shared.managed_directories\\[0\\].group_writable',
+    ):
+        create_shared_deploy_artifacts(
+            config={
+                'shared': {
+                    'managed_directories': [
+                        {
+                            'path': 'shared/writable',
+                            'group_writable': 'maybe',
+                        }
+                    ]
+                }
+            },
+            runtime={},
+            repo_root=str(tmp_path),
+            load_script_paths=[],
+            logger=_logger(),
+        )
+
+
 def test_create_shared_deploy_artifacts_requires_single_load_script(
     tmp_path: Path,
 ):