Support fsspec specifiers for ImageBuilder envs

Addresses #42

Author: pont-us

docs/xcetool.md

@@ -6,8 +6,8 @@ The command-line interface to xcengine is the command `xcetool`, which
 implements multiple subcommands and options for building and running
 container images and Application Packages.
 
-You can use the `--help` flag for any `xcetool` command or subcommand to get more
-details on usage and available options.
+You can use the `--help` flag for any `xcetool` command or subcommand to get
+more  details on usage and available options.
 
 ### `xcetool image build`
 
@@ -16,9 +16,9 @@ Usage: `xcetool image build [OPTIONS] NOTEBOOK`
 This is the main `xcetool` subcommand: it builds a container image from a
 supplied notebook and environment file. If given the `--eoap` argument, it also
 generates  a CWL file defining a corresponding application package. The
-NOTEBOOK argument can be a path to a local file, a URL, or any other string
-which can be parsed by the [fsspec](https://filesystem-spec.readthedocs.io/)
-library.
+NOTEBOOK argument can be a path to a local file, an HTTP URL, or any other
+string which can be parsed by the
+[fsspec](https://filesystem-spec.readthedocs.io/) library.
 
 Options:
 
@@ -28,6 +28,8 @@ Options:
     This option is mainly useful for debugging.
 -   `-e`, `--environment` `FILE`:
     Conda environment file to use in Docker image.
+    This can be a path to a local file, an HTTP URL, or any other string which
+    can be parsed by the fsspec library.
     If no environment file is specified here or in the notebook,
     xcetool will look for a file called `environment.yml`
     in the notebook's directory. If all else fails,

test/test_core.py

@@ -380,24 +380,41 @@ def test_image_builder_write_dockerfile(tmp_path):
 
 
 @patch("docker.from_env")
-@pytest.mark.parametrize("set_env", [False, True])
+@pytest.mark.parametrize("env_type", ["none", "local", "http"])
 @pytest.mark.parametrize("skip_build", [False, True])
-def test_image_builder_build_dir(from_env_mock, tmp_path, set_env, skip_build):
+def test_image_builder_build_dir(
+        from_env_mock,
+        tmp_path,
+        httpserver,
+        env_type,
+        skip_build
+):
     client_mock = Mock(docker.client.DockerClient)
     client_mock.images.build.return_value = None, None
     from_env_mock.return_value = client_mock
 
     build_dir = tmp_path / "build"
-    env_path = tmp_path / "env2.yaml"
+    build_env_path = tmp_path / "env2.yaml"
     env_def = {
         "name": "foo",
         "channels": "bar",
         "dependencies": ["python >=3.13", "baz >=42.0"],
     }
-    env_path.write_text(yaml.safe_dump(env_def))
+    build_env_path.write_text(yaml.safe_dump(env_def))
+    env_http = "/env2.yaml"
+
+    match env_type:
+        case "none": env_param = None
+        case "local": env_param = build_env_path
+        case "http":
+            httpserver.expect_request(env_http).respond_with_data(build_env_path.read_bytes())
+            env_param = httpserver.url_for(env_http)
+        case _:
+            raise RuntimeError(f"Unknown env type {env_type}")
+
     image_builder = ImageBuilder(
         pathlib.Path(__file__).parent / "data" / "noparamtest.ipynb",
-        env_path if set_env else None,
+        env_param,
         build_dir,
         None,
     )
@@ -406,11 +423,11 @@ def test_image_builder_build_dir(from_env_mock, tmp_path, set_env, skip_build):
         from_env_mock.assert_not_called()
     else:
         client_mock.images.build.assert_called()
-    env_path = build_dir / "environment.yml"
-    assert env_path.is_file()
-    output_env = yaml.safe_load(env_path.read_text())
+    build_env_path = build_dir / "environment.yml"
+    assert build_env_path.is_file()
+    output_env = yaml.safe_load(build_env_path.read_text())
     assert {"name", "channels", "dependencies"} <= set(output_env)
-    if set_env:
+    if env_type != "none":
         assert output_env["name"] == env_def["name"]
         assert output_env["channels"] == env_def["channels"]
         assert set(output_env["dependencies"]) >= set(env_def["dependencies"])

xcengine/core.py

@@ -187,12 +187,12 @@ class ImageBuilder:
     """
 
     tag_format: ClassVar[str] = "%Y.%m.%d.%H.%M.%S"
-    environment: pathlib.Path | None = None
+    environment: pathlib.Path | str | None = None
 
     def __init__(
         self,
-        notebook: pathlib.Path,
-        environment: pathlib.Path | None,
+        notebook: pathlib.Path | str,
+        environment: pathlib.Path | str | None,
         build_dir: pathlib.Path,
         tag: str | None,
     ):
@@ -244,7 +244,7 @@ def build(
     ) -> Image | None:
         self.script_creator.convert_notebook_to_script(self.build_dir)
         if self.environment:
-            with open(self.environment, "r") as fh:
+            with fsspec.open(self.environment, "r") as fh:
                 env_def = yaml.safe_load(fh)
         else:
             LOGGER.warning(