??? MSDOE

Author: syntron

OMPython/ModelicaSystem.py

@@ -1705,11 +1705,11 @@ def run_doe():
             simargs={"override": {'stopTime': 1.0}},
         )
         doe_mod.prepare()
-        doe_dict = doe_mod.get_doe()
+        doe_def = doe_mod.get_doe_definition()
         doe_mod.simulate()
-        doe_sol = doe_mod.get_solutions()
+        doe_sol = doe_mod.get_doe_solutions()
 
-        # ... work with doe_df and doe_sol ...
+        # ... work with doe_def and doe_sol ...
 
 
     if __name__ == "__main__":
@@ -1723,64 +1723,64 @@ def run_doe():
 
     def __init__(
             self,
-            fileName: Optional[str | os.PathLike | pathlib.Path] = None,
+            fileName: Optional[str | os.PathLike | pathlib.Path | OMCPath] = None,
             modelName: Optional[str] = None,
             lmodel: Optional[list[str | tuple[str, str]]] = None,
             commandLineOptions: Optional[str] = None,
             variableFilter: Optional[str] = None,
-            customBuildDirectory: Optional[str | os.PathLike | pathlib.Path] = None,
+            customBuildDirectory: Optional[str | os.PathLike | pathlib.Path | OMCPath] = None,
             omhome: Optional[str] = None,
             omc_process: Optional[OMCProcessLocal] = None,
 
-            simargs: Optional[dict[str, Optional[str | dict[str, str]]]] = None,
+            simargs: Optional[dict[str, Optional[str | dict[str, str] | numbers.Number]]] = None,
             timeout: Optional[int] = None,
 
             resultpath: Optional[pathlib.Path] = None,
             parameters: Optional[dict[str, list[str] | list[int] | list[float]]] = None,
+            # simulation_options
+            # ... other options to be set? ...
     ) -> None:
         """
         Initialisation of ModelicaSystemDoE. The parameters are based on: ModelicaSystem.__init__() and
         ModelicaSystem.simulate(). Additionally, the path to store the result files is needed (= resultpath) as well as
         a list of parameters to vary for the Doe (= parameters). All possible combinations are considered.
         """
         self._lmodel = lmodel
-        self._modelName = modelName
-        self._fileName = fileName
+        self._model_name = modelName
+        self._file_name = fileName
 
-        self._CommandLineOptions = commandLineOptions
-        self._variableFilter = variableFilter
-        self._customBuildDirectory = customBuildDirectory
+        self._command_line_options = commandLineOptions
+        self._variable_filter = variableFilter
+        self._custom_build_directory = customBuildDirectory
         self._omhome = omhome
         self._omc_process = omc_process
 
         # reference for the model; not used for any simulations but to evaluate parameters, etc.
         self._mod = ModelicaSystem(
-            fileName=self._fileName,
-            modelName=self._modelName,
+            fileName=self._file_name,
+            modelName=self._model_name,
             lmodel=self._lmodel,
-            commandLineOptions=self._CommandLineOptions,
-            variableFilter=self._variableFilter,
-            customBuildDirectory=self._customBuildDirectory,
+            commandLineOptions=self._command_line_options,
+            variableFilter=self._variable_filter,
+            customBuildDirectory=self._custom_build_directory,
             omhome=self._omhome,
             omc_process=self._omc_process,
         )
 
         self._simargs = simargs
         self._timeout = timeout
 
-        # TODO: Update her & all path handling of this class
-        if isinstance(resultpath, pathlib.Path):
-            self._resultpath = resultpath
-        else:
-            self._resultpath = pathlib.Path('.')
+        if resultpath is None:  # isinstance(resultpath, (OMCPath, pathlib.path, str, os.PathLike)):
+            raise ModelicaSystemError("Argument resultpath must be set to a valid path where the results are stored")
+        self._resultpath = self._mod._getconn.omcpath(resultpath)
 
         if isinstance(parameters, dict):
             self._parameters = parameters
         else:
             self._parameters = {}
 
-        self._sim_dict: Optional[dict[str, dict[str, Any]]] = None
-        self._sim_task_query: queue.Queue = queue.Queue()
+        self._doe_def: Optional[dict[str, dict[str, Any]]] = None
+        self._doe_cmd: Optional[dict[str, OMCSessionRunData]] = None
 
     def prepare(self) -> int:
         """
@@ -1790,6 +1790,8 @@ def prepare(self) -> int:
         The return value is the number of simulation defined.
         """
 
+        doe_sim = {}
+
         param_structure = {}
         param_simple = {}
         for param_name in self._parameters.keys():
@@ -1804,15 +1806,18 @@ def prepare(self) -> int:
         param_structure_combinations = list(itertools.product(*param_structure.values()))
         param_simple_combinations = list(itertools.product(*param_simple.values()))
 
-        self._sim_dict = {}
+        self._doe_def = {}
         for idx_pc_structure, pc_structure in enumerate(param_structure_combinations):
+            mod_structure_dir = self._mod.getWorkDirectory() / f"DOE_{idx_pc_structure:09d}"
+            mod_structure_dir.mkdir()
+
             mod_structure = ModelicaSystem(
-                fileName=self._fileName,
-                modelName=self._modelName,
+                fileName=self._file_name,
+                modelName=self._model_name,
                 lmodel=self._lmodel,
-                commandLineOptions=self._CommandLineOptions,
-                variableFilter=self._variableFilter,
-                customBuildDirectory=self._customBuildDirectory,
+                commandLineOptions=self._command_line_options,
+                variableFilter=self._variable_filter,
+                customBuildDirectory=mod_structure_dir,
                 omhome=self._omhome,
                 omc_process=self._omc_process,
                 build=False,
@@ -1824,20 +1829,21 @@ def prepare(self) -> int:
 
                 pk_value = pc_structure[idx_structure]
                 if isinstance(pk_value, str):
-                    expression = f"setParameterValue({self._modelName}, {pk_structure}, \"{pk_value}\")"
+                    expression = f"setParameterValue({self._model_name}, {pk_structure}, \"{pk_value}\")"
                 elif isinstance(pk_value, bool):
                     pk_value_bool_str = "true" if pk_value else "false"
-                    expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value_bool_str});"
+                    expression = f"setParameterValue({self._model_name}, {pk_structure}, {pk_value_bool_str});"
                 else:
-                    expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})"
+                    expression = f"setParameterValue({self._model_name}, {pk_structure}, {pk_value})"
                 res = mod_structure.sendExpression(expression)
                 if not res:
-                    raise ModelicaSystemError(f"Cannot set structural parameter {self._modelName}.{pk_structure} "
+                    raise ModelicaSystemError(f"Cannot set structural parameter {self._model_name}.{pk_structure} "
                                               f"to {pk_value} using {repr(expression)}")
 
-            mod_structure.buildModel(variableFilter=self._variableFilter)
+            mod_structure.buildModel(variableFilter=self._variable_filter)
 
             for idx_pc_simple, pc_simple in enumerate(param_simple_combinations):
+                # TODO: check mypy error message / type hints
                 sim_param_simple = {}
                 for idx_simple, pk_simple in enumerate(param_simple.keys()):
                     sim_param_simple[pk_simple] = pc_simple[idx_simple]
@@ -1862,23 +1868,41 @@ def prepare(self) -> int:
                         }
                 )
 
-                self._sim_dict[resfilename] = df_data
+                self._doe_def[resfilename] = df_data
+
+                mod_structure.setParameters(sim_param_simple)
 
                 mscmd = mod_structure.simulate_cmd(
-                    result_file=resultfile.absolute().resolve(),
+                    result_file=resultfile,
                     timeout=self._timeout,
                 )
                 if self._simargs is not None:
                     mscmd.args_set(args=self._simargs)
-                mscmd.args_set(args={"override": sim_param_simple})
+                # mscmd.args_set(args={"override": sim_param_simple})
+
+                cmd_definition = mscmd.definition()
+
+                # TODO: fix path to executable; copy it using OMCPath.copy() => no clone; only one OMC!
+                #       exe = exe.parent / (DOE name)- exe.name
+
+                doe_sim[resultfile.name] = cmd_definition
 
-                self._sim_task_query.put(mscmd)
+                del mscmd
 
-        logger.info(f"Prepared {self._sim_task_query.qsize()} simulation definitions for the defined DoE.")
+            logger.info(f'Status OMC: {mod_structure._getconn.sendExpression("getVersion()")}')
 
-        return self._sim_task_query.qsize()
+            # cleanup; ensure that there is not too much RAM consumption
+            logger.warning("DEL")
+            del mod_structure._getconn
+            del mod_structure
 
-    def get_doe(self) -> Optional[dict[str, dict[str, Any]]]:
+        logger.info(f"Prepared {len(doe_sim)} simulation definitions for the defined DoE.")
+
+        self._doe_cmd = doe_sim
+
+        return len(doe_sim)
+
+    def get_doe_definition(self) -> Optional[dict[str, dict[str, Any]]]:
         """
         Get the defined DoE as a dict, where each key is the result filename and the value is a dict of simulation
         settings including structural and non-structural parameters.
@@ -1888,12 +1912,18 @@ def get_doe(self) -> Optional[dict[str, dict[str, Any]]]:
         ```
         import pandas as pd
 
-        doe_dict = doe_mod.get_doe()
+        doe_dict = doe_mod.get_doe_definition()
         doe_df = pd.DataFrame.from_dict(data=doe_dict, orient='index')
         ```
 
         """
-        return self._sim_dict
+        return self._doe_def
+
+    def get_doe_command(self) -> Optional[dict[str, OMCSessionRunData]]:
+        """
+        Get the definitions of simulations commands to run for this DoE.
+        """
+        return self._doe_cmd
 
     def simulate(
             self,
@@ -1905,71 +1935,82 @@ def simulate(
         Returns True if all simulations were done successfully, else False.
         """
 
-        sim_query_total = self._sim_task_query.qsize()
-        if not isinstance(self._sim_dict, dict) or len(self._sim_dict) == 0:
+        doe_cmd_total = len(self._doe_cmd)
+        doe_def_total = len(self._doe_def)
+
+        if doe_cmd_total != doe_def_total:
+            raise ModelicaSystemError(f"Mismatch between number simulation commands ({doe_cmd_total}) "
+                                      f"and simulation definitions ({doe_def_total}).")
+
+        doe_task_query: queue.Queue = queue.Queue()
+        for doe_cmd in self._doe_cmd.values():
+            doe_task_query.put(doe_cmd)
+
+        if not isinstance(self._doe_def, dict) or len(self._doe_def) == 0:
             raise ModelicaSystemError("Missing Doe Summary!")
-        sim_dict_total = len(self._sim_dict)
 
         def worker(worker_id, task_queue):
             while True:
                 try:
                     # Get the next task from the queue
-                    mscmd = task_queue.get(block=False)
+                    cmd_definition = task_queue.get(block=False)
                 except queue.Empty:
-                    logger.info(f"[Worker {worker_id}] No more simulations to run.")
+                    logger.warning(f"[Worker {worker_id}] No more simulations to run.")
                     break
 
-                if mscmd is None:
+                if cmd_definition is None:
                     raise ModelicaSystemError("Missing simulation definition!")
 
-                resultfile = mscmd.arg_get(key='r')
-                resultpath = pathlib.Path(resultfile)
+                resultfile = cmd_definition.cmd_result_path
+                resultpath = self._mod._getconn.omcpath(resultfile)
 
-                logger.info(f"[Worker {worker_id}] Performing task: {resultpath.name}")
+                logger.warning(f"[Worker {worker_id}] Performing task: {resultpath.name}")
 
                 try:
-                    mscmd.run()
+                    returncode = self._mod._getconn.run_model_executable(cmd_run_data=cmd_definition)
+                    logger.warning(f"[Worker {worker_id}] Simulation {resultpath.name} "
+                                   f"finished with return code: {returncode}")
                 except ModelicaSystemError as ex:
                     logger.warning(f"Simulation error for {resultpath.name}: {ex}")
 
                 # Mark the task as done
                 task_queue.task_done()
 
-                sim_query_done = sim_query_total - self._sim_task_query.qsize()
-                logger.info(f"[Worker {worker_id}] Task completed: {resultpath.name} "
-                            f"({sim_query_total - sim_query_done}/{sim_query_total} = "
-                            f"{(sim_query_total - sim_query_done) / sim_query_total * 100:.2f}% of tasks left)")
+                sim_query_done = doe_cmd_total - doe_task_query.qsize()
+                logger.warning(f"[Worker {worker_id}] Task completed: {resultpath.name} "
+                               f"({doe_cmd_total - sim_query_done}/{doe_cmd_total} = "
+                               f"{(doe_cmd_total - sim_query_done) / doe_cmd_total * 100:.2f}% of tasks left)")
 
-        logger.info(f"Start simulations for DoE with {sim_query_total} simulations "
-                    f"using {num_workers} workers ...")
+        logger.warning(f"Start simulations for DoE with {doe_cmd_total} simulations "
+                       f"using {num_workers} workers ...")
 
         # Create and start worker threads
         threads = []
         for i in range(num_workers):
-            thread = threading.Thread(target=worker, args=(i, self._sim_task_query))
+            thread = threading.Thread(target=worker, args=(i, doe_task_query))
             thread.start()
             threads.append(thread)
 
         # Wait for all threads to complete
         for thread in threads:
             thread.join()
 
-        sim_dict_done = 0
-        for resultfilename in self._sim_dict:
+        doe_def_done = 0
+        for resultfilename in self._doe_def:
             resultfile = self._resultpath / resultfilename
 
             # include check for an empty (=> 0B) result file which indicates a crash of the model executable
             # see: https://github.com/OpenModelica/OMPython/issues/261
             # https://github.com/OpenModelica/OpenModelica/issues/13829
-            if resultfile.is_file() and resultfile.stat().st_size > 0:
-                self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE] = True
-                sim_dict_done += 1
+            if resultfile.is_file() and resultfile.size() > 0:
+                self._doe_def[resultfilename][self.DICT_RESULT_AVAILABLE] = True
+                doe_def_done += 1
 
-        logger.info(f"All workers finished ({sim_dict_done} of {sim_dict_total} simulations with a result file).")
+        logger.warning(f"All workers finished ({doe_def_done} of {doe_def_total} simulations with a result file).")
 
-        return sim_dict_total == sim_dict_done
+        return doe_def_total == doe_def_done
 
-    def get_solutions(
+    def get_doe_solutions(
             self,
             var_list: Optional[list] = None,
     ) -> Optional[tuple[str] | dict[str, dict[str, np.ndarray]]]:
@@ -1995,19 +2036,19 @@ def get_solutions(
         ```
 
         """
-        if not isinstance(self._sim_dict, dict):
+        if not isinstance(self._doe_def, dict):
             return None
 
-        if len(self._sim_dict) == 0:
+        if len(self._doe_def) == 0:
             raise ModelicaSystemError("No result files available - all simulations did fail?")
 
         sol_dict: dict[str, dict[str, Any]] = {}
-        for resultfilename in self._sim_dict:
+        for resultfilename in self._doe_def:
             resultfile = self._resultpath / resultfilename
 
             sol_dict[resultfilename] = {}
 
-            if not self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE]:
+            if not self._doe_def[resultfilename][self.DICT_RESULT_AVAILABLE]:
                 sol_dict[resultfilename]['msg'] = 'No result file available!'
                 sol_dict[resultfilename]['data'] = {}
                 continue