diff --git a/test/test_core.py b/test/test_core.py index 29347af..09153c1 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -354,7 +354,9 @@ def test_script_creator_notebook_config(): def test_script_creator_notebook_config_http(httpserver): http_path = "/mynotebook.ipynb" nb_path = pathlib.Path(__file__).parent / "data" / "paramtest.ipynb" - httpserver.expect_request(http_path).respond_with_data(nb_path.read_bytes()) + httpserver.expect_request(http_path).respond_with_data( + nb_path.read_bytes() + ) script_creator = ScriptCreator(httpserver.url_for(http_path)) config = script_creator.nb_params.config assert config["environment_file"] == "my-environment.yml" @@ -382,12 +384,20 @@ def test_image_builder_write_dockerfile(tmp_path): @patch("docker.from_env") @pytest.mark.parametrize("env_type", ["none", "local", "http"]) @pytest.mark.parametrize("skip_build", [False, True]) +@pytest.mark.parametrize("with_eoap", [False, True]) +@pytest.mark.parametrize("with_xcube", [False, True]) +@pytest.mark.parametrize("pystac_in_deps", [False, True]) +@pytest.mark.parametrize("xcube_in_deps", [False, True]) def test_image_builder_build_dir( - from_env_mock, - tmp_path, - httpserver, - env_type, - skip_build + from_env_mock, + tmp_path, + httpserver, + env_type, + skip_build, + with_eoap, + with_xcube, + pystac_in_deps, + xcube_in_deps, ): client_mock = Mock(docker.client.DockerClient) client_mock.images.build.return_value = None, None @@ -398,16 +408,22 @@ def test_image_builder_build_dir( env_def = { "name": "foo", "channels": "bar", - "dependencies": ["python >=3.13", "baz >=42.0"], + "dependencies": ["python >=3.13", "baz >=42.0"] + + (["pystac"] if pystac_in_deps else []) + + (["xcube"] if xcube_in_deps else []), } build_env_path.write_text(yaml.safe_dump(env_def)) env_http = "/env2.yaml" match env_type: - case "none": env_param = None - case "local": env_param = build_env_path + case "none": + env_param = None + case "local": + env_param = build_env_path case "http": - httpserver.expect_request(env_http).respond_with_data(build_env_path.read_bytes()) + httpserver.expect_request(env_http).respond_with_data( + build_env_path.read_bytes() + ) env_param = httpserver.url_for(env_http) case _: raise RuntimeError(f"Unknown env type {env_type}") @@ -418,7 +434,9 @@ def test_image_builder_build_dir( build_dir, None, ) - image_builder.build(skip_build=skip_build) + image_builder.build( + skip_build=skip_build, with_eoap=with_eoap, with_xcube=with_xcube + ) if skip_build: from_env_mock.assert_not_called() else: @@ -430,7 +448,12 @@ def test_image_builder_build_dir( if env_type != "none": assert output_env["name"] == env_def["name"] assert output_env["channels"] == env_def["channels"] - assert set(output_env["dependencies"]) >= set(env_def["dependencies"]) + + output_deps = set(output_env["dependencies"]) + input_deps = set(env_def["dependencies"]) + assert output_deps >= input_deps + assert ("pystac" in output_deps) == pystac_in_deps or with_eoap + assert ("xcube" in output_deps) == xcube_in_deps or with_xcube cwl = image_builder.create_cwl() assert "cwlVersion" in cwl diff --git a/test/test_util.py b/test/test_util.py index b50fedc..dd62852 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,3 +1,4 @@ +import argparse import logging import os import sys @@ -85,6 +86,61 @@ def test_write_stac(tmp_path, dataset, write_datasets, pre_existing_catalog): } + + + +def test_write_stac_no_pystac(tmp_path, dataset): + # Import hooks are the recommended "clean" way to do this, but don't work + # in this case. + orig_import = __import__ + def import_mock(name, *args): + if name == "pystac": + raise ModuleNotFoundError("No module named 'pystac'") + return orig_import(name, *args) + + with mock.patch("builtins.__import__", side_effect=import_mock): + # pytest imports xcengine.util long before we can patch __import__, + # so we delete pystac from util's namespace (if present) instead. + # This gives a NameError on access rather than a ModuleNotFoundError + # on import, but the important thing is to break any implementation + # that tries to import pystac without checking if it's available. + import xcengine.util + xcengine.util.__dict__.pop("pystac", None) + from xcengine.util import write_stac + write_stac({"ds1": dataset}, tmp_path) + # We want nothing to happen here, so no explicit assertions. + + +def test_start_server_no_xcube(dataset): + import io + orig_import = __import__ + def import_mock(name, *args): + if name == "xcube" or name.startswith("xcube."): + raise ModuleNotFoundError(f"No module named {name}") + return orig_import(name, *args) + + with mock.patch("builtins.__import__", side_effect=import_mock): + import xcengine.util + util_vars = ( + k for k in xcengine.util.__dict__.keys() + if k == "xcube" or k.startswith("xcube.") + ) + for v in util_vars: del xcengine.util.__dict__[v] + from xcengine.util import start_server + logger = logging.getLogger("test-start-server-logger") + logger.setLevel(logging.INFO) + logger.addHandler(logging.StreamHandler(log_stream := io.StringIO())) + start_server( + {"ds1": dataset}, + {}, + argparse.Namespace(batch=False, from_saved=False), + logger + ) + logged = log_stream.getvalue() + assert "Not starting" in logged + assert "Starting server" not in logged + + @pytest.mark.parametrize("eoap_mode", [False, True]) @pytest.mark.parametrize("ds2_format", [None, "zarr", "netcdf"]) def test_save_datasets(tmp_path, dataset, eoap_mode, ds2_format): diff --git a/xcengine/core.py b/xcengine/core.py index 5cbd315..f084b70 100644 --- a/xcengine/core.py +++ b/xcengine/core.py @@ -241,6 +241,8 @@ def __init__( def build( self, skip_build: bool = False, + with_eoap: bool = True, + with_xcube: bool = True, ) -> Image | None: self.script_creator.convert_notebook_to_script(self.build_dir) if self.environment: @@ -254,7 +256,11 @@ def build( ) env_def = self.export_conda_env() # We need xcube for server/viewer and pystac for EOAP stage-in/out - self.add_packages_to_environment(env_def, ["xcube", "pystac"]) + self.add_packages_to_environment( + env_def, + (["xcube"] if with_xcube else []) + + (["pystac"] if with_eoap else []), + ) with open(self.build_dir / "environment.yml", "w") as fh: fh.write(yaml.safe_dump(env_def)) self.write_dockerfile(self.build_dir / "Dockerfile") diff --git a/xcengine/util.py b/xcengine/util.py index 71dd2b4..2a470cb 100644 --- a/xcengine/util.py +++ b/xcengine/util.py @@ -1,18 +1,17 @@ # Copyright (c) 2024-2026 by Brockmann Consult GmbH # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. - +import argparse +import logging from datetime import datetime import json import pathlib import shutil from typing import NamedTuple, Mapping -import pystac import xarray as xr from xarray import Dataset - def clear_directory(directory: pathlib.Path) -> None: for path in directory.iterdir(): if path.is_dir(): @@ -24,6 +23,12 @@ def clear_directory(directory: pathlib.Path) -> None: def write_stac( datasets: Mapping[str, xr.Dataset], stac_root: pathlib.Path ) -> None: + try: + import pystac + except ModuleNotFoundError: + # If pystac isn't present, we assume that stage-out is not required + # and exit quietly. + return catalog_path = stac_root / "catalog.json" if catalog_path.exists(): # Assume that the user code generated its own stage-out data @@ -124,11 +129,21 @@ def save_datasets( return saved_datasets -def start_server(datasets, saved_datasets, args, logger): - import xcube.util.plugin - import xcube.webapi.viewer - from xcube.server.server import Server - from xcube.server.framework import get_framework_class +def start_server( + datasets: Mapping[str, xr.Dataset], + saved_datasets: dict[str, pathlib.Path], + args: argparse.Namespace, + logger: logging.Logger +): + try: + import xcube.util.plugin + import xcube.webapi.viewer + from xcube.server.server import Server + from xcube.server.framework import get_framework_class + except ImportError as e: + logger.info(e.msg) + logger.info("Not starting server, since xcube not available") + return xcube.util.plugin.init_plugins() server = Server(framework=get_framework_class("tornado")(), config={}) diff --git a/xcengine/wrapper.py b/xcengine/wrapper.py index 17ea2ea..6bca37c 100644 --- a/xcengine/wrapper.py +++ b/xcengine/wrapper.py @@ -38,12 +38,8 @@ def __xce_set_params(config_locator: str | pathlib.Path = sys.argv[0]): exec(user_code) import argparse -import pathlib import xarray as xr -import xcube.util.plugin -import xcube.core.new -import xcube.webapi.viewer def main(): @@ -62,7 +58,6 @@ def main(): if args.verbose > 0: LOGGER.setLevel(logging.DEBUG) - xcube.util.plugin.init_plugins() datasets = { name: thing for name, thing in globals().copy().items()