Add functionality to easily load output files from previous executions into a running app (#58)

Author: heidi-holm-4ss

dev-requirements.txt

@@ -5,4 +5,5 @@ isort
 pydata-sphinx-theme == 0.11.0
 pytest
 pytest-cov
-sphinx == 5.3.0
+sphinx == 5.3.0
+myst-parser<2.0

docs/api_ref/resultcollector.rst

@@ -12,4 +12,5 @@ ResultCollector
     :toctree:
     :template: class.rst
 
+    load_previous_engineroom_results
     ResultCollector

docs/user_guide/concepts_utils/collect_results.rst

@@ -1,5 +1,30 @@
 Collect and store results
 =========================
+Aapplication results can be stored inside a dedicated folder, so that the application output is easily available after app execution
+
+Application results can be any set of files (or files hierarchy), that is produced by your application, for example output CSV files
+containing processed data. Results are moved to a permanent store after every execution, regardless of exit code.
+
+Output files are appended to the result set from previous execution. If your application creates a file with the same name, it will
+override the previous result file. If you need to append to a file from a previous execution, the file must be loaded into the 
+application first. This can be achieved using :meth:`~fourinsight.engineroom.utils.load_previous_engineroom_results`, which will
+load the results from the permanent store into the application.
+
+
+.. code-block:: python
+
+    from fourinsight.api import UserSession
+    from fourinsight.engineroom.utils import load_previous_engineroom_results
+
+    session = UserSession()
+
+    #download all available results
+    load_previous_engineroom_results(ENGINE_ROOM_APP_ID, session, download_all=True)
+
+    #download specific results file
+    load_previous_engineroom_results(ENGINE_ROOM_APP_ID, session, path="config.json")
+
+
 The :class:`~fourinsight.engineroom.utils.ResultCollector` is a useful tool when you want to collect and store results.
 The basic usage is illustrated with the examples below.
 

fourinsight/engineroom/utils/__init__.py

@@ -7,6 +7,7 @@
     PersistentDict,
     PersistentJSON,
     ResultCollector,
+    load_previous_engineroom_results,
 )
 from ._datamanage import (
     BaseDataSource,
@@ -36,4 +37,5 @@
     "PersistentDict",
     "PersistentJSON",
     "ResultCollector",
+    "load_previous_engineroom_results",
 ]

fourinsight/engineroom/utils/_constants.py

@@ -0,0 +1 @@
+API_BASE_URL = "https://api.4insight.io"

fourinsight/engineroom/utils/_core.py

@@ -1,4 +1,6 @@
 import json
+import urllib.parse
+import warnings
 from abc import abstractmethod
 from collections.abc import MutableMapping
 from io import BytesIO, TextIOWrapper
@@ -8,6 +10,8 @@
 from azure.core.exceptions import ResourceNotFoundError
 from azure.storage.blob import BlobClient
 
+from ._constants import API_BASE_URL
+
 
 class BaseHandler(TextIOWrapper):
     """
@@ -548,3 +552,73 @@ def truncate(self, before=None, after=None):
             index_drop.extend(self._dataframe.index[(self._dataframe.index > after)])
         if index_drop:
             self.delete_rows(index_drop)
+
+
+def _get_all_previous_file_names(app_id, session):
+    """query all available results file from the EngineRoom application. Returns list of dicts"""
+    response = session.get(f"{API_BASE_URL}/v1.0/Applications/{app_id}/results")
+    response.raise_for_status()
+    results = response.json()
+    if not results:
+        warnings.warn(f"No results found for application ID {app_id}.", UserWarning)
+    return results
+
+
+def _build_download_url(app_id, navigable_file_name):
+    safe_name = urllib.parse.quote(navigable_file_name)
+    return f"{API_BASE_URL}/v1.0/Applications/{app_id}/results/{safe_name}/download"
+
+
+def _download_and_save_file(session, download_url, save_path):
+    save_path.parent.mkdir(parents=True, exist_ok=True)
+    response = session.get(download_url)
+    response.raise_for_status()
+    with open(save_path, "wb") as f:
+        f.write(response.content)
+
+
+def load_previous_engineroom_results(
+    app_id, session, path=None, download_all=False, output_folder="output"
+):
+    """
+    Load past EngineRoom results from a specified application and
+    store locally in the same output folder
+
+    Parameters
+    ----------
+    app_id : str
+        The EngineRoom application ID.
+    session : 4insight session object
+        Authorized 4insight session.
+    path : str or Path, optional
+        The file path within the EngineRoom output folder.
+        Ignored if download_all is True.
+    download_all : bool, optional
+        If True, download all results in the output folder. Defaults to False.
+    output_folder : str, optional
+        Name of the EngineRoom output folder. Defaults to "output".
+
+    """
+    output_folder = Path(output_folder)
+    available_results = _get_all_previous_file_names(app_id, session)
+    if not available_results:
+        return
+
+    available_file_names = [file["fileName"] for file in available_results]
+    navigable_file_names = [file["navigableFileName"] for file in available_results]
+
+    if download_all:
+        for file_name, nav_name in zip(available_file_names, navigable_file_names):
+            file_path = output_folder / file_name
+            download_url = _build_download_url(app_id, nav_name)
+            _download_and_save_file(session, download_url, file_path)
+    else:
+        if path not in available_file_names:
+            warnings.warn(
+                f"{path} not found in application {app_id} results.", UserWarning
+            )
+        else:
+            idx = available_file_names.index(path)
+            file_path = output_folder / path
+            download_url = _build_download_url(app_id, navigable_file_names[idx])
+            _download_and_save_file(session, download_url, file_path)

setup.cfg

@@ -22,6 +22,7 @@ install_requires =
     azure-storage-blob >= 12.4.0
     pyarrow
     numpy
+    fourinsight-api
 
 [options.packages.find]
 include =

tests/test_core.py

@@ -1,7 +1,9 @@
 import json
+import urllib.parse
+import warnings
 from io import BytesIO, TextIOWrapper
 from pathlib import Path
-from unittest.mock import Mock, patch
+from unittest.mock import ANY, MagicMock, Mock, mock_open, patch
 
 import numpy as np
 import pandas as pd
@@ -15,9 +17,16 @@
     NullHandler,
     PersistentDict,
     ResultCollector,
+    load_previous_engineroom_results,
+)
+from fourinsight.engineroom.utils._core import (
+    _build_download_url,
+    _download_and_save_file,
+    _get_all_previous_file_names,
 )
 
 REMOTE_FILE_PATH = Path(__file__).parent / "testdata/a_test_file.json"
+API_BASE_URL = "https://api.4insight.io"
 
 
 @pytest.fixture
@@ -55,6 +64,25 @@ def azure_blob_handler_mocked(mock_from_connection_string):
     return handler
 
 
+@pytest.fixture
+def previous_file_names():
+    filenames = [
+        {
+            "fileName": "config.json",
+            "navigableFileName": "config.json",
+            "safeName": "config.json",
+        },
+        {
+            "fileName": "SN00569 - 23AT/2024-12-02_131637/sensor_info/sensor_info.csv",
+            "navigableFileName": "SN00569 - 23AT*2024-12-02_131637*sensor_info*sensor_info.csv",
+            "safeName": urllib.parse.quote(
+                "SN00569 - 23AT*2024-12-02_131637*sensor_info*sensor_info.csv"
+            ),
+        },
+    ]
+    return filenames
+
+
 class Test_BaseHandler:
     def test__init__(self):
         handler = BaseHandler()
@@ -1210,3 +1238,147 @@ def test_delete_rows_truncate_int_both_none(self):
         ).astype({"a": "float64", "b": "string", "c": "Int64", "d": "float64"})
 
         pd.testing.assert_frame_equal(df_out, df_expect)
+
+
+def test__build_download_url(previous_file_names):
+    app_id = "12345"
+    for i in range(len(previous_file_names)):
+        navigable_filename = previous_file_names[i]["navigableFileName"]
+        safe_name = previous_file_names[i]["safeName"]
+        url = _build_download_url(app_id, navigable_filename)
+        assert (
+            url
+            == f"{API_BASE_URL}/v1.0/Applications/{app_id}/results/{safe_name}/download"
+        )
+
+
+class Test__download_and_save_file:
+    def setup_method(self):
+        # Common mocks
+        self.mock_session = MagicMock()
+        self.mock_response = MagicMock()
+        self.mock_response.content = b"this is the files"
+        self.mock_session.get.return_value = self.mock_response
+
+        self.url = "https://4insight.io/engineroom/result1.csv"
+        self.path = Path("output/results1.csv")
+
+    @patch("fourinsight.engineroom.utils._core.open", new_callable=mock_open)
+    @patch.object(Path, "mkdir")
+    def test_download_success(self, mock_mkdir, mock_file):
+        _download_and_save_file(self.mock_session, self.url, self.path)
+
+        self.mock_session.get.assert_called_once_with(self.url)
+        self.mock_response.raise_for_status.assert_called_once()
+        mock_file.assert_called_once_with(self.path, "wb")
+        mock_file().write.assert_called_once_with(b"this is the files")
+        mock_mkdir.assert_called_once_with(parents=True, exist_ok=True)
+
+    @patch("fourinsight.engineroom.utils._core.open", new_callable=mock_open)
+    @patch.object(Path, "mkdir")
+    def test_raises_exception_on_http_error(self, mock_mkdir, mock_file):
+        self.mock_response.raise_for_status.side_effect = RuntimeError("HTTP error")
+
+        with pytest.raises(RuntimeError, match="HTTP error"):
+            _download_and_save_file(self.mock_session, self.url, self.path)
+
+    @patch("fourinsight.engineroom.utils._core.open", new_callable=mock_open)
+    @patch.object(Path, "mkdir")
+    def test_creates_directory_if_missing(self, mock_mkdir, mock_file):
+        _download_and_save_file(self.mock_session, self.url, self.path)
+        mock_mkdir.assert_called_once_with(parents=True, exist_ok=True)
+
+
+class Test__get_all_previous_file_names:
+    def setup_method(self):
+        # Common mocks
+        self.mock_session = MagicMock()
+        self.mock_response = MagicMock()
+
+        self.mock_response.raise_for_status.return_value = None
+        self.mock_session.get.return_value = self.mock_response
+
+        self.url = "https://4insight.io/engineroom/result1.csv"
+        self.path = Path("output/results1.csv")
+
+    def test_successful_response(self, previous_file_names):
+        self.mock_response.json.return_value = previous_file_names
+        app_id = "app123"
+        results = _get_all_previous_file_names(app_id, self.mock_session)
+        assert results == previous_file_names
+        self.mock_session.get.assert_called_once_with(
+            f"{API_BASE_URL}/v1.0/Applications/{app_id}/results"
+        )
+        self.mock_response.raise_for_status.assert_called_once()
+
+    def test_empty_results_returns_warning(self):
+        self.mock_response.json.return_value = []
+        app_id = "app123"
+        with pytest.warns(
+            UserWarning, match=f"No results found for application ID {app_id}."
+        ):
+            _get_all_previous_file_names(app_id, self.mock_session)
+
+
+class Test_load_previous_engineroom_results:
+    def setup_method(self):
+        self.mock_session = MagicMock()
+        self.mock_response = MagicMock()
+
+        self.mock_response.raise_for_status.return_value = None
+        self.mock_session.get.return_value = self.mock_response
+
+        self.url = "https://4insight.io/engineroom/result1.csv"
+        self.path = Path("output/results1.csv")
+
+    @patch("fourinsight.engineroom.utils._core._download_and_save_file")
+    @patch("fourinsight.engineroom.utils._core._get_all_previous_file_names")
+    def test_download_all(
+        self,
+        mock__get_all_previous_file_names,
+        mock__download_and_save_file,
+        previous_file_names,
+    ):
+        mock__get_all_previous_file_names.return_value = previous_file_names
+
+        load_previous_engineroom_results("app123", self.mock_session, download_all=True)
+        assert mock__get_all_previous_file_names.call_count == 1
+        assert mock__download_and_save_file.call_count == len(previous_file_names)
+        for i in range(len(previous_file_names)):
+            mock__download_and_save_file.assert_any_call(
+                self.mock_session,
+                ANY,
+                Path("output") / previous_file_names[i]["fileName"],
+            )
+
+    @patch("fourinsight.engineroom.utils._core._download_and_save_file")
+    @patch("fourinsight.engineroom.utils._core._get_all_previous_file_names")
+    def test_download_single_file(
+        self,
+        mock__get_all_previous_file_names,
+        mock__download_and_save_file,
+        previous_file_names,
+    ):
+        mock__get_all_previous_file_names.return_value = previous_file_names
+
+        load_previous_engineroom_results(
+            "app123", self.mock_session, previous_file_names[1]["fileName"]
+        )
+        assert mock__get_all_previous_file_names.call_count == 1
+        mock__download_and_save_file.assert_called_once_with(
+            self.mock_session, ANY, Path("output") / previous_file_names[1]["fileName"]
+        )
+
+    @patch("fourinsight.engineroom.utils._core._get_all_previous_file_names")
+    def test_raise_when_file_not_found(
+        self, mock__get_all_previous_file_names, previous_file_names
+    ):
+        mock__get_all_previous_file_names.return_value = previous_file_names
+
+        with pytest.warns(
+            UserWarning,
+            match="missing_file.json not found in application app123 results.",
+        ):
+            load_previous_engineroom_results(
+                "app123", self.mock_session, "missing_file.json"
+            )