Add functionality to validate project groups in config

Author: abhaasgoyal

benchcab/config.py

@@ -91,6 +91,17 @@ def read_optional_key(config: dict):
             raise ValueError(msg)
         config["project"] = os.environ["PROJECT"]
 
+    # Directory List is obtained from Gadi Resources - https://opus.nci.org.au/display/Help/0.+Welcome+to+Gadi
+    data_dirs = ["/g/data", "/scratch"]
+    groups = list(
+        set([group for data_dir in data_dirs for group in os.listdir(data_dir)])
+    )
+
+    if config["project"] not in groups:
+        msg = f"User is not a member of project [{config['project']}]: Check if project key is correct"
+
+        raise ValueError(msg)
+
     if "realisations" in config:
         for r in config["realisations"]:
             r["name"] = r.get("name")
@@ -152,8 +163,8 @@ def read_config(config_path: str) -> dict:
     """
     # Read configuration file
     config = read_config_file(config_path)
-    # Populate configuration dict with optional keys
-    read_optional_key(config)
     # Validate and return.
     validate_config(config)
+    # Populate configuration dict with optional keys
+    read_optional_key(config)
     return config

benchcab/data/test/config-optional.yml

@@ -1,5 +1,5 @@
 # Config with optional data
-project: optional
+project: hh5 
 
 fluxsite:
   experiment: AU-Tum

tests/test_config.py

@@ -13,7 +13,7 @@
 import benchcab.utils as bu
 
 # Temporarily set $PROJECT for testing module
-OPTIONAL_CONFIG_PROJECT = "tt1"
+OPTIONAL_CONFIG_PROJECT = "ks32"
 
 
 @pytest.fixture(autouse=True)
@@ -24,6 +24,13 @@ def _set_project_env_variable(monkeypatch):
         yield
 
 
+@pytest.fixture(autouse=True)
+def _set_project_validation_dirs():
+    with mock.patch("os.listdir") as mocked_listdir:
+        mocked_listdir.return_value = ["hh5", OPTIONAL_CONFIG_PROJECT]
+        yield
+
+
 @pytest.fixture()
 def config_str(request) -> str:
     """Provide relative YAML path string of data files."""
@@ -89,7 +96,7 @@ def all_optional_custom_config(default_only_config) -> dict:
     Reads from config-optional.yml
     """
     config = default_only_config | {
-        "project": "optional",
+        "project": "hh5",
         "fluxsite": {
             "experiment": "AU-Tum",
             "multiprocess": False,
@@ -163,15 +170,28 @@ def test_read_optional_key_add_data(input_config, output_config, request):
     assert pformat(config) == pformat(request.getfixturevalue(output_config))
 
 
-def test_no_project(default_only_config, monkeypatch):
+def test_no_project_name(default_only_config, monkeypatch):
     """If project key and $PROJECT are not provided, then raise error."""
     monkeypatch.delenv("PROJECT")
-    error_msg = re.escape(
+    err_msg = re.escape(
         """Couldn't resolve project: check 'project' in config.yaml
                 and/or $PROJECT set in ~/.config/gadi-login.conf
                 """
     )
-    with pytest.raises(ValueError, match=error_msg):
+    with pytest.raises(ValueError, match=err_msg):
+        bc.read_optional_key(default_only_config)
+
+
+def test_user_not_in_project(default_only_config):
+    """If user is not in viewable NCI projects, raise error."""
+    default_only_config["project"] = "non_existing"
+    err_msg = re.escape(
+        "User is not a member of project [non_existing]: Check if project key is correct"
+    )
+    with pytest.raises(
+        ValueError,
+        match=err_msg,
+    ):
         bc.read_optional_key(default_only_config)